2024-11-20 21:13:51,123 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-20 21:13:51,138 main DEBUG Took 0.012974 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-20 21:13:51,139 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-20 21:13:51,139 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-20 21:13:51,141 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-20 21:13:51,142 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:13:51,152 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-20 21:13:51,165 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:13:51,167 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:13:51,167 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:13:51,168 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:13:51,168 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:13:51,169 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:13:51,170 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:13:51,170 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:13:51,171 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:13:51,171 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:13:51,172 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:13:51,172 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:13:51,172 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:13:51,173 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:13:51,173 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:13:51,173 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:13:51,174 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:13:51,174 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:13:51,175 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:13:51,175 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:13:51,176 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:13:51,177 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:13:51,177 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:13:51,177 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 21:13:51,178 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:13:51,178 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-20 21:13:51,180 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 21:13:51,182 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-20 21:13:51,184 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-20 21:13:51,185 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-20 21:13:51,186 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-20 21:13:51,187 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-20 21:13:51,199 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-20 21:13:51,202 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-20 21:13:51,206 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-20 21:13:51,206 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-20 21:13:51,207 main DEBUG createAppenders(={Console}) 2024-11-20 21:13:51,208 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-20 21:13:51,208 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-20 21:13:51,208 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-20 21:13:51,209 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-20 21:13:51,210 main DEBUG OutputStream closed 2024-11-20 21:13:51,210 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-20 21:13:51,210 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-20 21:13:51,211 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-20 21:13:51,304 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-20 21:13:51,306 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-20 21:13:51,307 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-20 21:13:51,309 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-20 21:13:51,309 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-20 21:13:51,310 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-20 21:13:51,310 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-20 21:13:51,311 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-20 21:13:51,311 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-20 21:13:51,311 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-20 21:13:51,311 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-20 21:13:51,312 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-20 21:13:51,312 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-20 21:13:51,312 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-20 21:13:51,313 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-20 21:13:51,313 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-20 21:13:51,313 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-20 21:13:51,314 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-20 21:13:51,316 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20 21:13:51,317 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-20 21:13:51,317 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-20 21:13:51,318 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-20T21:13:51,603 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7 2024-11-20 21:13:51,606 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-20 21:13:51,607 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20T21:13:51,615 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-20T21:13:51,632 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-20T21:13:51,647 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7/cluster_6fcb2757-7fac-998c-5e18-650f39eceb78, deleteOnExit=true 2024-11-20T21:13:51,647 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-20T21:13:51,648 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7/test.cache.data in system properties and HBase conf 2024-11-20T21:13:51,648 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7/hadoop.tmp.dir in system properties and HBase conf 2024-11-20T21:13:51,649 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7/hadoop.log.dir in system properties and HBase conf 2024-11-20T21:13:51,650 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-20T21:13:51,650 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-20T21:13:51,651 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-20T21:13:51,734 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-20T21:13:51,829 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T21:13:51,833 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-20T21:13:51,834 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-20T21:13:51,835 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-20T21:13:51,835 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T21:13:51,836 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-20T21:13:51,837 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-20T21:13:51,837 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T21:13:51,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T21:13:51,838 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-20T21:13:51,839 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7/nfs.dump.dir in system properties and HBase conf 2024-11-20T21:13:51,840 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7/java.io.tmpdir in system properties and HBase conf 2024-11-20T21:13:51,840 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T21:13:51,841 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-20T21:13:51,841 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-20T21:13:52,843 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-20T21:13:52,946 INFO [Time-limited test {}] log.Log(170): Logging initialized @2627ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-20T21:13:53,036 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T21:13:53,102 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T21:13:53,139 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T21:13:53,139 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T21:13:53,141 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T21:13:53,161 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T21:13:53,166 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7/hadoop.log.dir/,AVAILABLE} 2024-11-20T21:13:53,168 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T21:13:53,447 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7/java.io.tmpdir/jetty-localhost-34211-hadoop-hdfs-3_4_1-tests_jar-_-any-17334523829294588050/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T21:13:53,456 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:34211} 2024-11-20T21:13:53,457 INFO [Time-limited test {}] server.Server(415): Started @3139ms 2024-11-20T21:13:53,885 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T21:13:53,893 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T21:13:53,894 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T21:13:53,894 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T21:13:53,895 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-20T21:13:53,896 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42946cd0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7/hadoop.log.dir/,AVAILABLE} 2024-11-20T21:13:53,896 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T21:13:54,036 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7acb84b5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7/java.io.tmpdir/jetty-localhost-42809-hadoop-hdfs-3_4_1-tests_jar-_-any-9436326416915792748/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T21:13:54,037 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6e1ebae6{HTTP/1.1, (http/1.1)}{localhost:42809} 2024-11-20T21:13:54,037 INFO [Time-limited test {}] server.Server(415): Started @3720ms 2024-11-20T21:13:54,118 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T21:13:54,638 WARN [Thread-71 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7/cluster_6fcb2757-7fac-998c-5e18-650f39eceb78/data/data1/current/BP-394376-172.17.0.3-1732137232539/current, will proceed with Du for space computation calculation, 2024-11-20T21:13:54,645 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7/cluster_6fcb2757-7fac-998c-5e18-650f39eceb78/data/data2/current/BP-394376-172.17.0.3-1732137232539/current, will proceed with Du for space computation calculation, 2024-11-20T21:13:54,747 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T21:13:54,818 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7 2024-11-20T21:13:54,824 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1d58e4ef046ed82 with lease ID 0x9d6ebe51a6142c79: Processing first storage report for DS-bd0379f2-d258-48a9-80cf-b36cb995539e from datanode DatanodeRegistration(127.0.0.1:42371, datanodeUuid=9cda7d19-7c5a-4207-a46c-89245f61bcba, infoPort=38183, infoSecurePort=0, ipcPort=33507, storageInfo=lv=-57;cid=testClusterID;nsid=541159662;c=1732137232539) 2024-11-20T21:13:54,826 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1d58e4ef046ed82 with lease ID 0x9d6ebe51a6142c79: from storage DS-bd0379f2-d258-48a9-80cf-b36cb995539e node DatanodeRegistration(127.0.0.1:42371, datanodeUuid=9cda7d19-7c5a-4207-a46c-89245f61bcba, infoPort=38183, infoSecurePort=0, ipcPort=33507, storageInfo=lv=-57;cid=testClusterID;nsid=541159662;c=1732137232539), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-20T21:13:54,827 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1d58e4ef046ed82 with lease ID 0x9d6ebe51a6142c79: Processing first storage report for DS-116c3f9a-746b-478e-a4e7-bf11cb9cdc26 from datanode DatanodeRegistration(127.0.0.1:42371, datanodeUuid=9cda7d19-7c5a-4207-a46c-89245f61bcba, infoPort=38183, infoSecurePort=0, ipcPort=33507, storageInfo=lv=-57;cid=testClusterID;nsid=541159662;c=1732137232539) 2024-11-20T21:13:54,827 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1d58e4ef046ed82 with lease ID 0x9d6ebe51a6142c79: from storage DS-116c3f9a-746b-478e-a4e7-bf11cb9cdc26 node DatanodeRegistration(127.0.0.1:42371, datanodeUuid=9cda7d19-7c5a-4207-a46c-89245f61bcba, infoPort=38183, infoSecurePort=0, ipcPort=33507, storageInfo=lv=-57;cid=testClusterID;nsid=541159662;c=1732137232539), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T21:13:54,920 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7/cluster_6fcb2757-7fac-998c-5e18-650f39eceb78/zookeeper_0, clientPort=57361, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7/cluster_6fcb2757-7fac-998c-5e18-650f39eceb78/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7/cluster_6fcb2757-7fac-998c-5e18-650f39eceb78/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T21:13:54,932 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57361 2024-11-20T21:13:54,946 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T21:13:54,950 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T21:13:55,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741825_1001 (size=7) 2024-11-20T21:13:55,642 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094 with version=8 2024-11-20T21:13:55,643 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/hbase-staging 2024-11-20T21:13:55,757 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-20T21:13:56,003 INFO [Time-limited test {}] client.ConnectionUtils(128): master/0b16ab3f13b7:0 server-side Connection retries=45 2024-11-20T21:13:56,013 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T21:13:56,014 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T21:13:56,018 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T21:13:56,018 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T21:13:56,019 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T21:13:56,172 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T21:13:56,246 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-20T21:13:56,257 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-20T21:13:56,261 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T21:13:56,293 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 2929 (auto-detected) 2024-11-20T21:13:56,294 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-20T21:13:56,316 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39687 2024-11-20T21:13:56,338 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39687 connecting to ZooKeeper ensemble=127.0.0.1:57361 2024-11-20T21:13:56,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:396870x0, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T21:13:56,375 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39687-0x1005f0abc6c0000 connected 2024-11-20T21:13:56,401 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T21:13:56,403 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T21:13:56,414 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39687-0x1005f0abc6c0000, quorum=127.0.0.1:57361, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T21:13:56,418 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094, hbase.cluster.distributed=false 2024-11-20T21:13:56,448 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39687-0x1005f0abc6c0000, quorum=127.0.0.1:57361, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T21:13:56,455 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39687 2024-11-20T21:13:56,456 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39687 2024-11-20T21:13:56,456 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39687 2024-11-20T21:13:56,460 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39687 2024-11-20T21:13:56,460 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39687 2024-11-20T21:13:56,591 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/0b16ab3f13b7:0 server-side Connection retries=45 2024-11-20T21:13:56,594 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T21:13:56,594 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T21:13:56,595 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T21:13:56,595 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T21:13:56,595 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T21:13:56,599 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T21:13:56,605 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T21:13:56,609 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36987 2024-11-20T21:13:56,612 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36987 connecting to ZooKeeper ensemble=127.0.0.1:57361 2024-11-20T21:13:56,614 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T21:13:56,626 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T21:13:56,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:369870x0, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T21:13:56,653 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36987-0x1005f0abc6c0001 connected 2024-11-20T21:13:56,654 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36987-0x1005f0abc6c0001, quorum=127.0.0.1:57361, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T21:13:56,660 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T21:13:56,673 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T21:13:56,677 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36987-0x1005f0abc6c0001, quorum=127.0.0.1:57361, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T21:13:56,686 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36987-0x1005f0abc6c0001, quorum=127.0.0.1:57361, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T21:13:56,696 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36987 2024-11-20T21:13:56,709 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36987 2024-11-20T21:13:56,717 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36987 2024-11-20T21:13:56,719 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36987 2024-11-20T21:13:56,719 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36987 2024-11-20T21:13:56,735 DEBUG [M:0;0b16ab3f13b7:39687 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;0b16ab3f13b7:39687 2024-11-20T21:13:56,736 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/0b16ab3f13b7,39687,1732137235812 2024-11-20T21:13:56,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36987-0x1005f0abc6c0001, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T21:13:56,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39687-0x1005f0abc6c0000, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T21:13:56,746 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39687-0x1005f0abc6c0000, quorum=127.0.0.1:57361, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/0b16ab3f13b7,39687,1732137235812 2024-11-20T21:13:56,766 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39687-0x1005f0abc6c0000, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:13:56,766 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36987-0x1005f0abc6c0001, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T21:13:56,766 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36987-0x1005f0abc6c0001, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:13:56,767 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39687-0x1005f0abc6c0000, quorum=127.0.0.1:57361, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T21:13:56,769 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/0b16ab3f13b7,39687,1732137235812 from backup master directory 2024-11-20T21:13:56,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39687-0x1005f0abc6c0000, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/0b16ab3f13b7,39687,1732137235812 2024-11-20T21:13:56,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36987-0x1005f0abc6c0001, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T21:13:56,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39687-0x1005f0abc6c0000, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T21:13:56,772 WARN [master/0b16ab3f13b7:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T21:13:56,772 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=0b16ab3f13b7,39687,1732137235812 2024-11-20T21:13:56,775 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-20T21:13:56,776 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-20T21:13:56,831 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/hbase.id] with ID: 2a21c2b7-ffb9-4430-8151-95213010e3ec 2024-11-20T21:13:56,831 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/.tmp/hbase.id 2024-11-20T21:13:56,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741826_1002 (size=42) 2024-11-20T21:13:57,249 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/.tmp/hbase.id]:[hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/hbase.id] 2024-11-20T21:13:57,304 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T21:13:57,310 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T21:13:57,330 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-20T21:13:57,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39687-0x1005f0abc6c0000, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:13:57,334 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36987-0x1005f0abc6c0001, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:13:57,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741827_1003 (size=196) 2024-11-20T21:13:57,371 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T21:13:57,374 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T21:13:57,394 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:13:57,398 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:13:57,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741828_1004 (size=1189) 2024-11-20T21:13:57,447 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/MasterData/data/master/store 2024-11-20T21:13:57,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741829_1005 (size=34) 2024-11-20T21:13:57,485 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-20T21:13:57,488 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:13:57,490 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T21:13:57,490 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T21:13:57,490 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T21:13:57,492 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T21:13:57,492 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T21:13:57,492 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T21:13:57,494 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732137237490Disabling compacts and flushes for region at 1732137237490Disabling writes for close at 1732137237492 (+2 ms)Writing region close event to WAL at 1732137237492Closed at 1732137237492 2024-11-20T21:13:57,501 WARN [master/0b16ab3f13b7:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/MasterData/data/master/store/.initializing 2024-11-20T21:13:57,501 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/MasterData/WALs/0b16ab3f13b7,39687,1732137235812 2024-11-20T21:13:57,519 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T21:13:57,540 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b16ab3f13b7%2C39687%2C1732137235812, suffix=, logDir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/MasterData/WALs/0b16ab3f13b7,39687,1732137235812, archiveDir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/MasterData/oldWALs, maxLogs=10 2024-11-20T21:13:57,574 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/MasterData/WALs/0b16ab3f13b7,39687,1732137235812/0b16ab3f13b7%2C39687%2C1732137235812.1732137237546, exclude list is [], retry=0 2024-11-20T21:13:57,599 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42371,DS-bd0379f2-d258-48a9-80cf-b36cb995539e,DISK] 2024-11-20T21:13:57,604 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-20T21:13:57,650 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/MasterData/WALs/0b16ab3f13b7,39687,1732137235812/0b16ab3f13b7%2C39687%2C1732137235812.1732137237546 2024-11-20T21:13:57,651 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38183:38183)] 2024-11-20T21:13:57,652 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:13:57,652 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:13:57,656 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:13:57,657 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:13:57,702 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:13:57,736 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T21:13:57,741 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:13:57,748 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:13:57,749 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:13:57,753 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T21:13:57,754 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:13:57,755 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:13:57,755 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:13:57,758 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T21:13:57,758 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:13:57,759 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:13:57,760 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:13:57,763 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T21:13:57,763 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:13:57,764 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:13:57,765 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:13:57,769 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:13:57,770 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:13:57,777 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:13:57,778 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:13:57,782 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T21:13:57,786 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T21:13:57,790 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T21:13:57,792 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72058531, jitterRate=0.07375578582286835}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T21:13:57,800 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732137237676Initializing all the Stores at 1732137237679 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732137237679Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732137237680 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732137237681 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732137237681Cleaning up temporary data from old regions at 1732137237778 (+97 ms)Region opened successfully at 1732137237800 (+22 ms) 2024-11-20T21:13:57,802 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T21:13:57,852 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54b8555, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0b16ab3f13b7/172.17.0.3:0 2024-11-20T21:13:57,896 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T21:13:57,914 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-20T21:13:57,914 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-20T21:13:57,919 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T21:13:57,921 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-20T21:13:57,928 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-20T21:13:57,928 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T21:13:57,971 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T21:13:57,981 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39687-0x1005f0abc6c0000, quorum=127.0.0.1:57361, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T21:13:57,983 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T21:13:57,986 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T21:13:57,987 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39687-0x1005f0abc6c0000, quorum=127.0.0.1:57361, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T21:13:57,988 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T21:13:57,991 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T21:13:57,994 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39687-0x1005f0abc6c0000, quorum=127.0.0.1:57361, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T21:13:57,996 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T21:13:57,997 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39687-0x1005f0abc6c0000, quorum=127.0.0.1:57361, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T21:13:57,999 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T21:13:58,019 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39687-0x1005f0abc6c0000, quorum=127.0.0.1:57361, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T21:13:58,021 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T21:13:58,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36987-0x1005f0abc6c0001, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T21:13:58,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39687-0x1005f0abc6c0000, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T21:13:58,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36987-0x1005f0abc6c0001, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:13:58,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39687-0x1005f0abc6c0000, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:13:58,033 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=0b16ab3f13b7,39687,1732137235812, sessionid=0x1005f0abc6c0000, setting cluster-up flag (Was=false) 2024-11-20T21:13:58,050 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39687-0x1005f0abc6c0000, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:13:58,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36987-0x1005f0abc6c0001, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:13:58,059 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T21:13:58,062 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0b16ab3f13b7,39687,1732137235812 2024-11-20T21:13:58,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39687-0x1005f0abc6c0000, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:13:58,069 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36987-0x1005f0abc6c0001, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:13:58,076 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T21:13:58,078 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=0b16ab3f13b7,39687,1732137235812 2024-11-20T21:13:58,086 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T21:13:58,125 INFO [RS:0;0b16ab3f13b7:36987 {}] regionserver.HRegionServer(746): ClusterId : 2a21c2b7-ffb9-4430-8151-95213010e3ec 2024-11-20T21:13:58,128 DEBUG [RS:0;0b16ab3f13b7:36987 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T21:13:58,134 DEBUG [RS:0;0b16ab3f13b7:36987 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T21:13:58,134 DEBUG [RS:0;0b16ab3f13b7:36987 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T21:13:58,138 DEBUG [RS:0;0b16ab3f13b7:36987 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T21:13:58,138 DEBUG [RS:0;0b16ab3f13b7:36987 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d97c7e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=0b16ab3f13b7/172.17.0.3:0 2024-11-20T21:13:58,160 DEBUG [RS:0;0b16ab3f13b7:36987 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;0b16ab3f13b7:36987 2024-11-20T21:13:58,165 INFO [RS:0;0b16ab3f13b7:36987 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T21:13:58,165 INFO [RS:0;0b16ab3f13b7:36987 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T21:13:58,166 DEBUG [RS:0;0b16ab3f13b7:36987 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T21:13:58,169 INFO [RS:0;0b16ab3f13b7:36987 {}] regionserver.HRegionServer(2659): reportForDuty to master=0b16ab3f13b7,39687,1732137235812 with port=36987, startcode=1732137236536 2024-11-20T21:13:58,180 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T21:13:58,185 DEBUG [RS:0;0b16ab3f13b7:36987 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T21:13:58,191 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T21:13:58,198 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T21:13:58,205 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 0b16ab3f13b7,39687,1732137235812 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T21:13:58,218 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/0b16ab3f13b7:0, corePoolSize=5, maxPoolSize=5 2024-11-20T21:13:58,218 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/0b16ab3f13b7:0, corePoolSize=5, maxPoolSize=5 2024-11-20T21:13:58,218 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/0b16ab3f13b7:0, corePoolSize=5, maxPoolSize=5 2024-11-20T21:13:58,218 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/0b16ab3f13b7:0, corePoolSize=5, maxPoolSize=5 2024-11-20T21:13:58,219 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/0b16ab3f13b7:0, corePoolSize=10, maxPoolSize=10 2024-11-20T21:13:58,219 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:13:58,219 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/0b16ab3f13b7:0, corePoolSize=2, maxPoolSize=2 2024-11-20T21:13:58,220 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:13:58,223 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732137268223 2024-11-20T21:13:58,226 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T21:13:58,226 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T21:13:58,227 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T21:13:58,228 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T21:13:58,234 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T21:13:58,234 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35349, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T21:13:58,234 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T21:13:58,234 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T21:13:58,235 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T21:13:58,236 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:13:58,236 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T21:13:58,243 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T21:13:58,247 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39687 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:13:58,248 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T21:13:58,250 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T21:13:58,250 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T21:13:58,256 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T21:13:58,257 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T21:13:58,265 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/0b16ab3f13b7:0:becomeActiveMaster-HFileCleaner.large.0-1732137238258,5,FailOnTimeoutGroup] 2024-11-20T21:13:58,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741831_1007 (size=1321) 2024-11-20T21:13:58,272 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/0b16ab3f13b7:0:becomeActiveMaster-HFileCleaner.small.0-1732137238266,5,FailOnTimeoutGroup] 2024-11-20T21:13:58,273 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T21:13:58,273 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T21:13:58,274 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T21:13:58,274 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094 2024-11-20T21:13:58,275 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T21:13:58,275 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T21:13:58,284 DEBUG [RS:0;0b16ab3f13b7:36987 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-20T21:13:58,285 WARN [RS:0;0b16ab3f13b7:36987 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-20T21:13:58,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741832_1008 (size=32) 2024-11-20T21:13:58,389 INFO [RS:0;0b16ab3f13b7:36987 {}] regionserver.HRegionServer(2659): reportForDuty to master=0b16ab3f13b7,39687,1732137235812 with port=36987, startcode=1732137236536 2024-11-20T21:13:58,394 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39687 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 0b16ab3f13b7,36987,1732137236536 2024-11-20T21:13:58,397 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39687 {}] master.ServerManager(517): Registering regionserver=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:13:58,408 DEBUG [RS:0;0b16ab3f13b7:36987 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094 2024-11-20T21:13:58,409 DEBUG [RS:0;0b16ab3f13b7:36987 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35549 2024-11-20T21:13:58,409 DEBUG [RS:0;0b16ab3f13b7:36987 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T21:13:58,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39687-0x1005f0abc6c0000, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T21:13:58,414 DEBUG [RS:0;0b16ab3f13b7:36987 {}] zookeeper.ZKUtil(111): regionserver:36987-0x1005f0abc6c0001, quorum=127.0.0.1:57361, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/0b16ab3f13b7,36987,1732137236536 2024-11-20T21:13:58,415 WARN [RS:0;0b16ab3f13b7:36987 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T21:13:58,415 INFO [RS:0;0b16ab3f13b7:36987 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:13:58,415 DEBUG [RS:0;0b16ab3f13b7:36987 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/WALs/0b16ab3f13b7,36987,1732137236536 2024-11-20T21:13:58,417 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [0b16ab3f13b7,36987,1732137236536] 2024-11-20T21:13:58,447 INFO [RS:0;0b16ab3f13b7:36987 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T21:13:58,468 INFO [RS:0;0b16ab3f13b7:36987 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T21:13:58,478 INFO [RS:0;0b16ab3f13b7:36987 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T21:13:58,478 INFO [RS:0;0b16ab3f13b7:36987 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T21:13:58,480 INFO [RS:0;0b16ab3f13b7:36987 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T21:13:58,488 INFO [RS:0;0b16ab3f13b7:36987 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T21:13:58,493 INFO [RS:0;0b16ab3f13b7:36987 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T21:13:58,494 DEBUG [RS:0;0b16ab3f13b7:36987 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:13:58,494 DEBUG [RS:0;0b16ab3f13b7:36987 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:13:58,495 DEBUG [RS:0;0b16ab3f13b7:36987 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:13:58,495 DEBUG [RS:0;0b16ab3f13b7:36987 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:13:58,495 DEBUG [RS:0;0b16ab3f13b7:36987 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:13:58,495 DEBUG [RS:0;0b16ab3f13b7:36987 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/0b16ab3f13b7:0, corePoolSize=2, maxPoolSize=2 2024-11-20T21:13:58,496 DEBUG [RS:0;0b16ab3f13b7:36987 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:13:58,496 DEBUG [RS:0;0b16ab3f13b7:36987 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:13:58,496 DEBUG [RS:0;0b16ab3f13b7:36987 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:13:58,496 DEBUG [RS:0;0b16ab3f13b7:36987 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:13:58,497 DEBUG [RS:0;0b16ab3f13b7:36987 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:13:58,497 DEBUG [RS:0;0b16ab3f13b7:36987 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/0b16ab3f13b7:0, corePoolSize=1, maxPoolSize=1 2024-11-20T21:13:58,497 DEBUG [RS:0;0b16ab3f13b7:36987 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/0b16ab3f13b7:0, corePoolSize=3, maxPoolSize=3 2024-11-20T21:13:58,498 DEBUG [RS:0;0b16ab3f13b7:36987 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0, corePoolSize=3, maxPoolSize=3 2024-11-20T21:13:58,505 INFO [RS:0;0b16ab3f13b7:36987 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T21:13:58,505 INFO [RS:0;0b16ab3f13b7:36987 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T21:13:58,505 INFO [RS:0;0b16ab3f13b7:36987 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T21:13:58,505 INFO [RS:0;0b16ab3f13b7:36987 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T21:13:58,506 INFO [RS:0;0b16ab3f13b7:36987 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T21:13:58,506 INFO [RS:0;0b16ab3f13b7:36987 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b16ab3f13b7,36987,1732137236536-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T21:13:58,537 INFO [RS:0;0b16ab3f13b7:36987 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T21:13:58,540 INFO [RS:0;0b16ab3f13b7:36987 {}] hbase.ChoreService(168): Chore ScheduledChore name=0b16ab3f13b7,36987,1732137236536-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T21:13:58,540 INFO [RS:0;0b16ab3f13b7:36987 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:13:58,541 INFO [RS:0;0b16ab3f13b7:36987 {}] regionserver.Replication(171): 0b16ab3f13b7,36987,1732137236536 started 2024-11-20T21:13:58,568 INFO [RS:0;0b16ab3f13b7:36987 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:13:58,568 INFO [RS:0;0b16ab3f13b7:36987 {}] regionserver.HRegionServer(1482): Serving as 0b16ab3f13b7,36987,1732137236536, RpcServer on 0b16ab3f13b7/172.17.0.3:36987, sessionid=0x1005f0abc6c0001 2024-11-20T21:13:58,569 DEBUG [RS:0;0b16ab3f13b7:36987 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T21:13:58,569 DEBUG [RS:0;0b16ab3f13b7:36987 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 0b16ab3f13b7,36987,1732137236536 2024-11-20T21:13:58,569 DEBUG [RS:0;0b16ab3f13b7:36987 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b16ab3f13b7,36987,1732137236536' 2024-11-20T21:13:58,569 DEBUG [RS:0;0b16ab3f13b7:36987 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T21:13:58,571 DEBUG [RS:0;0b16ab3f13b7:36987 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T21:13:58,572 DEBUG [RS:0;0b16ab3f13b7:36987 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T21:13:58,573 DEBUG [RS:0;0b16ab3f13b7:36987 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T21:13:58,573 DEBUG [RS:0;0b16ab3f13b7:36987 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 0b16ab3f13b7,36987,1732137236536 2024-11-20T21:13:58,573 DEBUG [RS:0;0b16ab3f13b7:36987 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '0b16ab3f13b7,36987,1732137236536' 2024-11-20T21:13:58,573 DEBUG [RS:0;0b16ab3f13b7:36987 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T21:13:58,577 DEBUG [RS:0;0b16ab3f13b7:36987 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T21:13:58,578 DEBUG [RS:0;0b16ab3f13b7:36987 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T21:13:58,578 INFO [RS:0;0b16ab3f13b7:36987 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T21:13:58,579 INFO [RS:0;0b16ab3f13b7:36987 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T21:13:58,686 INFO [RS:0;0b16ab3f13b7:36987 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T21:13:58,692 INFO [RS:0;0b16ab3f13b7:36987 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b16ab3f13b7%2C36987%2C1732137236536, suffix=, logDir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/WALs/0b16ab3f13b7,36987,1732137236536, archiveDir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/oldWALs, maxLogs=32 2024-11-20T21:13:58,696 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:13:58,701 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T21:13:58,705 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T21:13:58,706 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:13:58,707 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:13:58,707 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T21:13:58,710 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T21:13:58,710 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:13:58,711 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:13:58,711 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T21:13:58,715 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T21:13:58,715 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:13:58,716 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:13:58,716 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T21:13:58,720 DEBUG [RS:0;0b16ab3f13b7:36987 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/WALs/0b16ab3f13b7,36987,1732137236536/0b16ab3f13b7%2C36987%2C1732137236536.1732137238697, exclude list is [], retry=0 2024-11-20T21:13:58,720 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T21:13:58,721 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:13:58,723 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:13:58,723 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T21:13:58,725 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/hbase/meta/1588230740 2024-11-20T21:13:58,726 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/hbase/meta/1588230740 2024-11-20T21:13:58,730 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T21:13:58,730 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T21:13:58,731 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T21:13:58,735 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T21:13:58,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42371,DS-bd0379f2-d258-48a9-80cf-b36cb995539e,DISK] 2024-11-20T21:13:58,745 INFO [RS:0;0b16ab3f13b7:36987 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/WALs/0b16ab3f13b7,36987,1732137236536/0b16ab3f13b7%2C36987%2C1732137236536.1732137238697 2024-11-20T21:13:58,754 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T21:13:58,757 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70838616, jitterRate=0.055577635765075684}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T21:13:58,761 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732137238696Initializing all the Stores at 1732137238699 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732137238699Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732137238700 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732137238700Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732137238700Cleaning up temporary data from old regions at 1732137238730 (+30 ms)Region opened successfully at 1732137238760 (+30 ms) 2024-11-20T21:13:58,761 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T21:13:58,761 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T21:13:58,761 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T21:13:58,762 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T21:13:58,762 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T21:13:58,773 DEBUG [RS:0;0b16ab3f13b7:36987 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38183:38183)] 2024-11-20T21:13:58,776 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T21:13:58,777 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732137238761Disabling compacts and flushes for region at 1732137238761Disabling writes for close at 1732137238762 (+1 ms)Writing region close event to WAL at 1732137238776 (+14 ms)Closed at 1732137238776 2024-11-20T21:13:58,781 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T21:13:58,781 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T21:13:58,797 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T21:13:58,812 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T21:13:58,815 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T21:13:58,969 DEBUG [0b16ab3f13b7:39687 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-20T21:13:58,988 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:13:58,997 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0b16ab3f13b7,36987,1732137236536, state=OPENING 2024-11-20T21:13:59,037 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T21:13:59,045 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39687-0x1005f0abc6c0000, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:13:59,046 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36987-0x1005f0abc6c0001, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:13:59,047 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T21:13:59,048 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T21:13:59,051 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=0b16ab3f13b7,36987,1732137236536}] 2024-11-20T21:13:59,055 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T21:13:59,233 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T21:13:59,237 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52809, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T21:13:59,247 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T21:13:59,247 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T21:13:59,248 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T21:13:59,251 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=0b16ab3f13b7%2C36987%2C1732137236536.meta, suffix=.meta, logDir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/WALs/0b16ab3f13b7,36987,1732137236536, archiveDir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/oldWALs, maxLogs=32 2024-11-20T21:13:59,267 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/WALs/0b16ab3f13b7,36987,1732137236536/0b16ab3f13b7%2C36987%2C1732137236536.meta.1732137239253.meta, exclude list is [], retry=0 2024-11-20T21:13:59,270 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42371,DS-bd0379f2-d258-48a9-80cf-b36cb995539e,DISK] 2024-11-20T21:13:59,273 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/WALs/0b16ab3f13b7,36987,1732137236536/0b16ab3f13b7%2C36987%2C1732137236536.meta.1732137239253.meta 2024-11-20T21:13:59,273 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38183:38183)] 2024-11-20T21:13:59,273 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:13:59,275 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T21:13:59,278 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T21:13:59,282 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T21:13:59,287 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T21:13:59,287 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:13:59,287 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T21:13:59,287 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T21:13:59,290 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T21:13:59,292 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T21:13:59,292 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:13:59,292 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:13:59,293 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T21:13:59,294 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T21:13:59,294 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:13:59,295 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:13:59,295 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T21:13:59,296 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T21:13:59,296 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:13:59,297 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:13:59,297 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T21:13:59,299 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T21:13:59,299 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:13:59,299 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T21:13:59,300 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T21:13:59,301 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/hbase/meta/1588230740 2024-11-20T21:13:59,304 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/hbase/meta/1588230740 2024-11-20T21:13:59,306 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T21:13:59,306 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T21:13:59,307 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T21:13:59,310 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T21:13:59,312 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68648960, jitterRate=0.02294921875}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T21:13:59,312 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T21:13:59,315 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732137239288Writing region info on filesystem at 1732137239288Initializing all the Stores at 1732137239290 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732137239290Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732137239290Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732137239290Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732137239290Cleaning up temporary data from old regions at 1732137239306 (+16 ms)Running coprocessor post-open hooks at 1732137239312 (+6 ms)Region opened successfully at 1732137239315 (+3 ms) 2024-11-20T21:13:59,322 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732137239222 2024-11-20T21:13:59,334 DEBUG [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T21:13:59,335 INFO [RS_OPEN_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T21:13:59,337 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:13:59,339 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 0b16ab3f13b7,36987,1732137236536, state=OPEN 2024-11-20T21:13:59,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36987-0x1005f0abc6c0001, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T21:13:59,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39687-0x1005f0abc6c0000, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T21:13:59,345 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T21:13:59,345 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T21:13:59,345 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:13:59,350 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T21:13:59,351 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=0b16ab3f13b7,36987,1732137236536 in 295 msec 2024-11-20T21:13:59,357 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T21:13:59,357 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 555 msec 2024-11-20T21:13:59,358 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T21:13:59,358 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T21:13:59,379 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:13:59,380 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:13:59,401 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:13:59,403 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53767, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:13:59,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2990 sec 2024-11-20T21:13:59,425 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732137239425, completionTime=-1 2024-11-20T21:13:59,428 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-20T21:13:59,428 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T21:13:59,455 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-20T21:13:59,455 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732137299455 2024-11-20T21:13:59,455 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732137359455 2024-11-20T21:13:59,455 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 27 msec 2024-11-20T21:13:59,458 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b16ab3f13b7,39687,1732137235812-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T21:13:59,458 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b16ab3f13b7,39687,1732137235812-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:13:59,458 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b16ab3f13b7,39687,1732137235812-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:13:59,459 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-0b16ab3f13b7:39687, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:13:59,460 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T21:13:59,460 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T21:13:59,466 DEBUG [master/0b16ab3f13b7:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T21:13:59,487 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.714sec 2024-11-20T21:13:59,488 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T21:13:59,490 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T21:13:59,491 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T21:13:59,491 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T21:13:59,491 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T21:13:59,492 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b16ab3f13b7,39687,1732137235812-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T21:13:59,493 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b16ab3f13b7,39687,1732137235812-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T21:13:59,501 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T21:13:59,501 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T21:13:59,502 INFO [master/0b16ab3f13b7:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=0b16ab3f13b7,39687,1732137235812-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T21:13:59,536 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@132591a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:13:59,538 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-20T21:13:59,538 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-20T21:13:59,541 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:13:59,544 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:13:59,553 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:13:59,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:13:59,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:13:59,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@519651b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:13:59,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:13:59,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:13:59,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:13:59,566 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43638, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:13:59,569 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39c236f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:13:59,570 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:13:59,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:13:59,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:13:59,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:13:59,608 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=0b16ab3f13b7,39687,1732137235812 2024-11-20T21:13:59,624 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=209, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=515, ProcessCount=11, AvailableMemoryMB=1643 2024-11-20T21:13:59,635 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T21:13:59,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 0b16ab3f13b7,39687,1732137235812 2024-11-20T21:13:59,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@a1209f1 2024-11-20T21:13:59,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T21:13:59,656 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T21:13:59,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T21:13:59,695 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T21:13:59,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T21:13:59,716 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T21:13:59,719 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:13:59,720 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-20T21:13:59,721 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T21:13:59,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T21:13:59,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741835_1011 (size=1061) 2024-11-20T21:13:59,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T21:14:00,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T21:14:00,160 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094 2024-11-20T21:14:00,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741836_1012 (size=53) 2024-11-20T21:14:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T21:14:00,577 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:14:00,578 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f3e2abb3d075e2d6a5c2948af99b8c71, disabling compactions & flushes 2024-11-20T21:14:00,578 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:00,578 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:00,578 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. after waiting 0 ms 2024-11-20T21:14:00,578 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:00,578 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:00,578 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f3e2abb3d075e2d6a5c2948af99b8c71: Waiting for close lock at 1732137240577Disabling compacts and flushes for region at 1732137240577Disabling writes for close at 1732137240578 (+1 ms)Writing region close event to WAL at 1732137240578Closed at 1732137240578 2024-11-20T21:14:00,580 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T21:14:00,585 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732137240580"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732137240580"}]},"ts":"1732137240580"} 2024-11-20T21:14:00,590 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T21:14:00,592 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T21:14:00,594 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732137240592"}]},"ts":"1732137240592"} 2024-11-20T21:14:00,598 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T21:14:00,600 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3e2abb3d075e2d6a5c2948af99b8c71, ASSIGN}] 2024-11-20T21:14:00,602 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3e2abb3d075e2d6a5c2948af99b8c71, ASSIGN 2024-11-20T21:14:00,604 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3e2abb3d075e2d6a5c2948af99b8c71, ASSIGN; state=OFFLINE, location=0b16ab3f13b7,36987,1732137236536; forceNewPlan=false, retain=false 2024-11-20T21:14:00,756 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f3e2abb3d075e2d6a5c2948af99b8c71, regionState=OPENING, regionLocation=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:00,761 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3e2abb3d075e2d6a5c2948af99b8c71, ASSIGN because future has completed 2024-11-20T21:14:00,762 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536}] 2024-11-20T21:14:00,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T21:14:00,957 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:00,957 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:14:00,958 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:00,958 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:14:00,959 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:00,959 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:00,997 INFO [StoreOpener-f3e2abb3d075e2d6a5c2948af99b8c71-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:01,003 INFO [StoreOpener-f3e2abb3d075e2d6a5c2948af99b8c71-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T21:14:01,003 INFO [StoreOpener-f3e2abb3d075e2d6a5c2948af99b8c71-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3e2abb3d075e2d6a5c2948af99b8c71 columnFamilyName A 2024-11-20T21:14:01,003 DEBUG [StoreOpener-f3e2abb3d075e2d6a5c2948af99b8c71-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:01,005 INFO [StoreOpener-f3e2abb3d075e2d6a5c2948af99b8c71-1 {}] regionserver.HStore(327): Store=f3e2abb3d075e2d6a5c2948af99b8c71/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:14:01,005 INFO [StoreOpener-f3e2abb3d075e2d6a5c2948af99b8c71-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:01,008 INFO [StoreOpener-f3e2abb3d075e2d6a5c2948af99b8c71-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T21:14:01,008 INFO [StoreOpener-f3e2abb3d075e2d6a5c2948af99b8c71-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3e2abb3d075e2d6a5c2948af99b8c71 columnFamilyName B 2024-11-20T21:14:01,009 DEBUG [StoreOpener-f3e2abb3d075e2d6a5c2948af99b8c71-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:01,014 INFO [StoreOpener-f3e2abb3d075e2d6a5c2948af99b8c71-1 {}] regionserver.HStore(327): Store=f3e2abb3d075e2d6a5c2948af99b8c71/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:14:01,015 INFO [StoreOpener-f3e2abb3d075e2d6a5c2948af99b8c71-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:01,018 INFO [StoreOpener-f3e2abb3d075e2d6a5c2948af99b8c71-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T21:14:01,019 INFO [StoreOpener-f3e2abb3d075e2d6a5c2948af99b8c71-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f3e2abb3d075e2d6a5c2948af99b8c71 columnFamilyName C 2024-11-20T21:14:01,019 DEBUG [StoreOpener-f3e2abb3d075e2d6a5c2948af99b8c71-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:01,020 INFO [StoreOpener-f3e2abb3d075e2d6a5c2948af99b8c71-1 {}] regionserver.HStore(327): Store=f3e2abb3d075e2d6a5c2948af99b8c71/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:14:01,021 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:01,021 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:01,023 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:01,024 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:01,026 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:01,026 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:01,027 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T21:14:01,030 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:01,050 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T21:14:01,051 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened f3e2abb3d075e2d6a5c2948af99b8c71; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72360877, jitterRate=0.07826109230518341}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T21:14:01,052 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:01,053 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for f3e2abb3d075e2d6a5c2948af99b8c71: Running coprocessor pre-open hook at 1732137240959Writing region info on filesystem at 1732137240959Initializing all the Stores at 1732137240974 (+15 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732137240974Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732137240984 (+10 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732137240984Cleaning up temporary data from old regions at 1732137241026 (+42 ms)Running coprocessor post-open hooks at 1732137241052 (+26 ms)Region opened successfully at 1732137241053 (+1 ms) 2024-11-20T21:14:01,061 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., pid=6, masterSystemTime=1732137240921 2024-11-20T21:14:01,082 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:01,082 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:01,086 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=f3e2abb3d075e2d6a5c2948af99b8c71, regionState=OPEN, openSeqNum=2, regionLocation=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:01,096 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39687 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=0b16ab3f13b7,36987,1732137236536, table=TestAcidGuarantees, region=f3e2abb3d075e2d6a5c2948af99b8c71. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T21:14:01,108 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 because future has completed 2024-11-20T21:14:01,125 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-20T21:14:01,125 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 in 357 msec 2024-11-20T21:14:01,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-20T21:14:01,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3e2abb3d075e2d6a5c2948af99b8c71, ASSIGN in 525 msec 2024-11-20T21:14:01,132 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T21:14:01,133 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732137241132"}]},"ts":"1732137241132"} 2024-11-20T21:14:01,138 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T21:14:01,141 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T21:14:01,148 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.4390 sec 2024-11-20T21:14:01,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T21:14:01,889 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:14:01,896 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bafc0a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:01,896 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:14:01,896 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:14:01,898 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:14:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:14:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:14:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ecb3032, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:01,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:14:01,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:14:01,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:01,902 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43660, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:14:01,904 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3922c1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:01,904 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:14:01,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:14:01,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:14:01,910 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:14:01,912 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T21:14:01,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 0b16ab3f13b7,39687,1732137235812 2024-11-20T21:14:01,914 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7eef39e4 2024-11-20T21:14:01,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T21:14:01,920 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T21:14:01,925 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@706bb323, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:01,925 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:14:01,925 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:14:01,926 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:14:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:14:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:14:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a183203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:01,927 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:14:01,928 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:14:01,928 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:01,930 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:14:01,931 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ef8cf77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:01,931 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a52ffa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:01,931 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:14:01,932 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:14:01,933 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:14:01,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:14:01,933 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:14:01,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ef1bd73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:01,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:14:01,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:14:01,936 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:01,937 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43692, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:14:01,939 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af2c0a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:01,939 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f14cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:01,940 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:14:01,940 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:14:01,941 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:14:01,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:14:01,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:14:01,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fc8436a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:01,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:14:01,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:14:01,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:01,944 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:14:01,945 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7230d8e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:01,946 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4201ea29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:01,946 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:14:01,946 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:14:01,947 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:14:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:14:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:14:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@553a6f47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:14:01,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:14:01,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:01,950 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:14:01,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eefee7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:01,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cc41cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:01,951 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:14:01,952 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:14:01,952 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:14:01,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:14:01,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:14:01,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2aa43e09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:01,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:14:01,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:14:01,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:01,954 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43736, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:14:01,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@792c9a75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:01,956 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b274eea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:01,956 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:14:01,957 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:14:01,957 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:14:01,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:14:01,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:14:01,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c197857, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:01,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:14:01,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:14:01,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:01,960 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:14:01,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b66dceb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:01,961 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d39686f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:01,961 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:14:01,962 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:14:01,962 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:14:01,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:14:01,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:14:01,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52e2fecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:01,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:14:01,963 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:14:01,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:01,964 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:14:01,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4501fb62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:01,968 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@761fbe54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:01,969 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:14:01,969 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:14:01,969 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:14:01,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:14:01,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:14:01,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1118bf09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:01,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:14:01,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:14:01,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:01,971 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:14:01,972 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@339e1ebf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:01,973 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9a31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:01,973 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:14:01,973 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:14:01,974 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:14:01,974 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:14:01,974 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:14:01,974 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38dcce6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:01,974 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:14:01,975 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:14:01,975 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:01,976 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:14:01,976 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9c26bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:01,980 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:14:01,981 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:14:01,981 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:14:01,981 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:14:01,982 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:14:01,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:14:01,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:14:01,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:14:01,983 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:14:01,983 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:14:01,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:14:01,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:14:01,985 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40978, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:14:01,985 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:14:01,985 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:14:01,986 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:14:01,990 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:14:01,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:14:01,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2] 2024-11-20T21:14:01,992 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2] 2024-11-20T21:14:01,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:14:01,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2] 2024-11-20T21:14:01,992 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:14:01,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2] 2024-11-20T21:14:01,997 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41012, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:14:01,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:14:01,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:14:02,000 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:14:02,001 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2] 2024-11-20T21:14:02,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2] 2024-11-20T21:14:02,018 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:14:02,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-20T21:14:02,026 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:14:02,029 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:14:02,029 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:14:02,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:14:02,033 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:14:02,033 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:14:02,035 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:14:02,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2] 2024-11-20T21:14:02,041 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:14:02,043 DEBUG [Thread-156 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:14:02,043 DEBUG [Thread-156 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:14:02,045 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41036, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:14:02,049 DEBUG [Thread-156 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2] 2024-11-20T21:14:02,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T21:14:02,064 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:14:02,067 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:14:02,068 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:14:02,070 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:14:02,073 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2] 2024-11-20T21:14:02,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:02,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T21:14:02,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3e2abb3d075e2d6a5c2948af99b8c71 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T21:14:02,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T21:14:02,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:02,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:02,236 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=A 2024-11-20T21:14:02,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:02,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=B 2024-11-20T21:14:02,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:02,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=C 2024-11-20T21:14:02,237 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:02,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:02,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:02,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:02,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:02,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T21:14:02,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T21:14:02,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:02,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:02,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:02,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:02,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:02,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:02,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:02,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:02,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:02,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137302465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:02,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137302468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:02,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137302470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:02,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:02,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137302473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:02,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:02,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:02,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:02,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:02,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:02,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:02,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:02,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:02,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:02,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:02,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:02,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137302476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:02,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:02,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:02,501 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:02,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/51d5a0c139024693af6992fbbc420165 is 50, key is test_row_0/A:col10/1732137242202/Put/seqid=0 2024-11-20T21:14:02,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741837_1013 (size=12001) 2024-11-20T21:14:02,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/51d5a0c139024693af6992fbbc420165 2024-11-20T21:14:02,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T21:14:02,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:02,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137302608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:02,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:02,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:02,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:02,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:02,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:02,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137302609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:02,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137302609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:02,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:02,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:02,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:02,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:02,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:02,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:02,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:02,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:02,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137302610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:02,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:02,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:02,620 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:02,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137302610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:02,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:02,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:02,622 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:02,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:02,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:02,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:02,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:02,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:02,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:02,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T21:14:02,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/ae2f1f399b174c5aac8432b0d0e887bb is 50, key is test_row_0/B:col10/1732137242202/Put/seqid=0 2024-11-20T21:14:02,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741838_1014 (size=12001) 2024-11-20T21:14:02,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/ae2f1f399b174c5aac8432b0d0e887bb 2024-11-20T21:14:02,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T21:14:02,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:02,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:02,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:02,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:02,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:02,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:02,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/9b35e56cec954e0ebf21d63658b0d5c9 is 50, key is test_row_0/C:col10/1732137242202/Put/seqid=0 2024-11-20T21:14:02,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:02,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:02,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137302830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:02,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137302829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:02,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:02,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:02,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:02,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:02,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137302830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:02,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:02,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:02,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:02,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:02,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:02,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:02,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137302831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:02,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:02,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:02,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:02,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:02,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:02,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137302837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:02,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:02,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:02,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:02,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741839_1015 (size=12001) 2024-11-20T21:14:02,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/9b35e56cec954e0ebf21d63658b0d5c9 2024-11-20T21:14:02,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/51d5a0c139024693af6992fbbc420165 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/51d5a0c139024693af6992fbbc420165 2024-11-20T21:14:02,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/51d5a0c139024693af6992fbbc420165, entries=150, sequenceid=16, filesize=11.7 K 2024-11-20T21:14:02,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/ae2f1f399b174c5aac8432b0d0e887bb as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/ae2f1f399b174c5aac8432b0d0e887bb 2024-11-20T21:14:02,900 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/ae2f1f399b174c5aac8432b0d0e887bb, entries=150, sequenceid=16, filesize=11.7 K 2024-11-20T21:14:02,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/9b35e56cec954e0ebf21d63658b0d5c9 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/9b35e56cec954e0ebf21d63658b0d5c9 2024-11-20T21:14:02,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/9b35e56cec954e0ebf21d63658b0d5c9, entries=150, sequenceid=16, filesize=11.7 K 2024-11-20T21:14:02,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for f3e2abb3d075e2d6a5c2948af99b8c71 in 731ms, sequenceid=16, compaction requested=false 2024-11-20T21:14:02,923 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-20T21:14:02,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:02,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-20T21:14:02,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:02,941 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing f3e2abb3d075e2d6a5c2948af99b8c71 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T21:14:02,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=A 2024-11-20T21:14:02,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:02,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=B 2024-11-20T21:14:02,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:02,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=C 2024-11-20T21:14:02,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:02,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/483e0ac434054a63b4ee1a1cc9527295 is 50, key is test_row_0/A:col10/1732137242472/Put/seqid=0 2024-11-20T21:14:02,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741840_1016 (size=12001) 2024-11-20T21:14:03,004 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/483e0ac434054a63b4ee1a1cc9527295 2024-11-20T21:14:03,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/dc26e315658649be831aefcf744b8066 is 50, key is test_row_0/B:col10/1732137242472/Put/seqid=0 2024-11-20T21:14:03,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:03,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:03,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741841_1017 (size=12001) 2024-11-20T21:14:03,165 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/dc26e315658649be831aefcf744b8066 2024-11-20T21:14:03,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T21:14:03,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/10d1eec311054a6b858b56c7fc0c9c0b is 50, key is test_row_0/C:col10/1732137242472/Put/seqid=0 2024-11-20T21:14:03,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:03,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137303216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,237 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:03,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:03,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137303220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:03,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:03,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137303227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:03,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:03,252 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:03,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137303234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:03,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741842_1018 (size=12001) 2024-11-20T21:14:03,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:03,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:03,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137303238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:03,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:03,260 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/10d1eec311054a6b858b56c7fc0c9c0b 2024-11-20T21:14:03,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/483e0ac434054a63b4ee1a1cc9527295 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/483e0ac434054a63b4ee1a1cc9527295 2024-11-20T21:14:03,310 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/483e0ac434054a63b4ee1a1cc9527295, entries=150, sequenceid=33, filesize=11.7 K 2024-11-20T21:14:03,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/dc26e315658649be831aefcf744b8066 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/dc26e315658649be831aefcf744b8066 2024-11-20T21:14:03,325 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/dc26e315658649be831aefcf744b8066, entries=150, sequenceid=33, filesize=11.7 K 2024-11-20T21:14:03,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/10d1eec311054a6b858b56c7fc0c9c0b as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/10d1eec311054a6b858b56c7fc0c9c0b 2024-11-20T21:14:03,345 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/10d1eec311054a6b858b56c7fc0c9c0b, entries=150, sequenceid=33, filesize=11.7 K 2024-11-20T21:14:03,349 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=93.93 KB/96180 for f3e2abb3d075e2d6a5c2948af99b8c71 in 407ms, sequenceid=33, compaction requested=false 2024-11-20T21:14:03,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:03,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:03,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-20T21:14:03,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-20T21:14:03,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-20T21:14:03,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3190 sec 2024-11-20T21:14:03,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.3420 sec 2024-11-20T21:14:03,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:03,384 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3e2abb3d075e2d6a5c2948af99b8c71 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-20T21:14:03,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=A 2024-11-20T21:14:03,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:03,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=B 2024-11-20T21:14:03,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:03,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=C 2024-11-20T21:14:03,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:03,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/ce8776eae5cc4c16a94ff44e02ebea2b is 50, key is test_row_0/A:col10/1732137243220/Put/seqid=0 2024-11-20T21:14:03,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741843_1019 (size=16681) 2024-11-20T21:14:03,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/ce8776eae5cc4c16a94ff44e02ebea2b 2024-11-20T21:14:03,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/daac3169ebcf42a5b8224b0d3972dcf8 is 50, key is test_row_0/B:col10/1732137243220/Put/seqid=0 2024-11-20T21:14:03,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:03,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137303418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:03,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:03,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137303423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:03,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:03,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137303428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:03,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:03,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,464 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:03,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:03,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137303457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:03,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:03,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:03,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137303461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:03,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:03,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741844_1020 (size=12001) 2024-11-20T21:14:03,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/daac3169ebcf42a5b8224b0d3972dcf8 2024-11-20T21:14:03,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/fcd83eca2d774303a260154cae1c5b26 is 50, key is test_row_0/C:col10/1732137243220/Put/seqid=0 2024-11-20T21:14:03,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741845_1021 (size=12001) 2024-11-20T21:14:03,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/fcd83eca2d774303a260154cae1c5b26 2024-11-20T21:14:03,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/ce8776eae5cc4c16a94ff44e02ebea2b as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/ce8776eae5cc4c16a94ff44e02ebea2b 2024-11-20T21:14:03,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/ce8776eae5cc4c16a94ff44e02ebea2b, entries=250, sequenceid=52, filesize=16.3 K 2024-11-20T21:14:03,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/daac3169ebcf42a5b8224b0d3972dcf8 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/daac3169ebcf42a5b8224b0d3972dcf8 2024-11-20T21:14:03,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/daac3169ebcf42a5b8224b0d3972dcf8, entries=150, sequenceid=52, filesize=11.7 K 2024-11-20T21:14:03,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:03,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137303573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:03,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,590 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:03,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/fcd83eca2d774303a260154cae1c5b26 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/fcd83eca2d774303a260154cae1c5b26 2024-11-20T21:14:03,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:03,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137303573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:03,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:03,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:03,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137303573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:03,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:03,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:03,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137303578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:03,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,594 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:03,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:03,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137303581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:03,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:03,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/fcd83eca2d774303a260154cae1c5b26, entries=150, sequenceid=52, filesize=11.7 K 2024-11-20T21:14:03,604 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for f3e2abb3d075e2d6a5c2948af99b8c71 in 221ms, sequenceid=52, compaction requested=true 2024-11-20T21:14:03,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:03,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3e2abb3d075e2d6a5c2948af99b8c71:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:14:03,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:03,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3e2abb3d075e2d6a5c2948af99b8c71:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:14:03,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:03,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3e2abb3d075e2d6a5c2948af99b8c71:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:14:03,625 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:03,625 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:03,628 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:03,629 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:03,631 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): f3e2abb3d075e2d6a5c2948af99b8c71/B is initiating minor compaction (all files) 2024-11-20T21:14:03,632 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3e2abb3d075e2d6a5c2948af99b8c71/B in TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:03,633 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/ae2f1f399b174c5aac8432b0d0e887bb, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/dc26e315658649be831aefcf744b8066, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/daac3169ebcf42a5b8224b0d3972dcf8] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp, totalSize=35.2 K 2024-11-20T21:14:03,634 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:03,634 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): f3e2abb3d075e2d6a5c2948af99b8c71/A is initiating minor compaction (all files) 2024-11-20T21:14:03,634 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3e2abb3d075e2d6a5c2948af99b8c71/A in TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:03,634 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/51d5a0c139024693af6992fbbc420165, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/483e0ac434054a63b4ee1a1cc9527295, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/ce8776eae5cc4c16a94ff44e02ebea2b] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp, totalSize=39.7 K 2024-11-20T21:14:03,635 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting ae2f1f399b174c5aac8432b0d0e887bb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732137242175 2024-11-20T21:14:03,635 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51d5a0c139024693af6992fbbc420165, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732137242175 2024-11-20T21:14:03,637 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 483e0ac434054a63b4ee1a1cc9527295, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732137242449 2024-11-20T21:14:03,637 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting dc26e315658649be831aefcf744b8066, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732137242449 2024-11-20T21:14:03,638 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting daac3169ebcf42a5b8224b0d3972dcf8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732137243214 2024-11-20T21:14:03,638 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce8776eae5cc4c16a94ff44e02ebea2b, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732137243214 2024-11-20T21:14:03,681 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3e2abb3d075e2d6a5c2948af99b8c71#B#compaction#9 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:03,681 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3e2abb3d075e2d6a5c2948af99b8c71#A#compaction#10 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:03,682 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/a2e6ac5c2ca74354bf47eb626718245d is 50, key is test_row_0/B:col10/1732137243220/Put/seqid=0 2024-11-20T21:14:03,683 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/b5e5077ca22a4974b765f82fcb07c3e4 is 50, key is test_row_0/A:col10/1732137243220/Put/seqid=0 2024-11-20T21:14:03,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741846_1022 (size=12104) 2024-11-20T21:14:03,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741847_1023 (size=12104) 2024-11-20T21:14:03,770 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/a2e6ac5c2ca74354bf47eb626718245d as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/a2e6ac5c2ca74354bf47eb626718245d 2024-11-20T21:14:03,770 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/b5e5077ca22a4974b765f82fcb07c3e4 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/b5e5077ca22a4974b765f82fcb07c3e4 2024-11-20T21:14:03,797 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3e2abb3d075e2d6a5c2948af99b8c71/A of f3e2abb3d075e2d6a5c2948af99b8c71 into b5e5077ca22a4974b765f82fcb07c3e4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:03,797 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:03,797 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3e2abb3d075e2d6a5c2948af99b8c71/B of f3e2abb3d075e2d6a5c2948af99b8c71 into a2e6ac5c2ca74354bf47eb626718245d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:03,797 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:03,803 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., storeName=f3e2abb3d075e2d6a5c2948af99b8c71/A, priority=13, startTime=1732137243606; duration=0sec 2024-11-20T21:14:03,803 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., storeName=f3e2abb3d075e2d6a5c2948af99b8c71/B, priority=13, startTime=1732137243624; duration=0sec 2024-11-20T21:14:03,804 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:03,804 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3e2abb3d075e2d6a5c2948af99b8c71:A 2024-11-20T21:14:03,804 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:03,806 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:03,806 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3e2abb3d075e2d6a5c2948af99b8c71:B 2024-11-20T21:14:03,807 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:03,807 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): f3e2abb3d075e2d6a5c2948af99b8c71/C is initiating minor compaction (all files) 2024-11-20T21:14:03,807 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3e2abb3d075e2d6a5c2948af99b8c71/C in TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:03,808 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/9b35e56cec954e0ebf21d63658b0d5c9, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/10d1eec311054a6b858b56c7fc0c9c0b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/fcd83eca2d774303a260154cae1c5b26] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp, totalSize=35.2 K 2024-11-20T21:14:03,808 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b35e56cec954e0ebf21d63658b0d5c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732137242175 2024-11-20T21:14:03,809 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10d1eec311054a6b858b56c7fc0c9c0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1732137242449 2024-11-20T21:14:03,812 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcd83eca2d774303a260154cae1c5b26, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732137243214 2024-11-20T21:14:03,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3e2abb3d075e2d6a5c2948af99b8c71 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-20T21:14:03,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:03,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=A 2024-11-20T21:14:03,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:03,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=B 2024-11-20T21:14:03,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:03,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=C 2024-11-20T21:14:03,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:03,843 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3e2abb3d075e2d6a5c2948af99b8c71#C#compaction#11 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:03,844 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/0320de669d494778b6333c6e5e95bb7d is 50, key is test_row_0/C:col10/1732137243220/Put/seqid=0 2024-11-20T21:14:03,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/b890b621823642d2bf0a83a6cc8ffd18 is 50, key is test_row_0/A:col10/1732137243450/Put/seqid=0 2024-11-20T21:14:03,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:03,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137303888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:03,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741848_1024 (size=12104) 2024-11-20T21:14:03,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:03,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:03,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137303897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:03,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:03,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137303900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:03,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:03,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:03,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:03,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137303902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:03,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137303907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:03,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:03,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:03,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:03,923 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/0320de669d494778b6333c6e5e95bb7d as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/0320de669d494778b6333c6e5e95bb7d 2024-11-20T21:14:03,936 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3e2abb3d075e2d6a5c2948af99b8c71/C of f3e2abb3d075e2d6a5c2948af99b8c71 into 0320de669d494778b6333c6e5e95bb7d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:03,936 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:03,936 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., storeName=f3e2abb3d075e2d6a5c2948af99b8c71/C, priority=13, startTime=1732137243624; duration=0sec 2024-11-20T21:14:03,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741849_1025 (size=12001) 2024-11-20T21:14:03,937 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:03,937 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3e2abb3d075e2d6a5c2948af99b8c71:C 2024-11-20T21:14:03,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/b890b621823642d2bf0a83a6cc8ffd18 2024-11-20T21:14:03,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/90d105a5f2d043a1b9646ed5c5c6e27d is 50, key is test_row_0/B:col10/1732137243450/Put/seqid=0 2024-11-20T21:14:03,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741850_1026 (size=12001) 2024-11-20T21:14:03,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/90d105a5f2d043a1b9646ed5c5c6e27d 2024-11-20T21:14:04,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/ffd2b6e61ce043f08bc4f1ce13685c6a is 50, key is test_row_0/C:col10/1732137243450/Put/seqid=0 2024-11-20T21:14:04,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:04,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137304018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:04,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,022 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:04,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:04,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137304018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:04,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:04,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:04,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137304019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:04,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:04,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:04,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137304028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741851_1027 (size=12001) 2024-11-20T21:14:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:04,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:04,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137304028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:04,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:04,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-20T21:14:04,208 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:14:04,209 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:14:04,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-20T21:14:04,213 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:14:04,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T21:14:04,215 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:14:04,215 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:14:04,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:04,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137304229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:04,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:04,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:04,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137304229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:04,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:04,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:04,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137304238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:04,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,249 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:04,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:04,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137304238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:04,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:04,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:04,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137304252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:04,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:04,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T21:14:04,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-20T21:14:04,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:04,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:04,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:04,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:04,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:04,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:04,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/ffd2b6e61ce043f08bc4f1ce13685c6a 2024-11-20T21:14:04,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/b890b621823642d2bf0a83a6cc8ffd18 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/b890b621823642d2bf0a83a6cc8ffd18 2024-11-20T21:14:04,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/b890b621823642d2bf0a83a6cc8ffd18, entries=150, sequenceid=69, filesize=11.7 K 2024-11-20T21:14:04,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/90d105a5f2d043a1b9646ed5c5c6e27d as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/90d105a5f2d043a1b9646ed5c5c6e27d 2024-11-20T21:14:04,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/90d105a5f2d043a1b9646ed5c5c6e27d, entries=150, sequenceid=69, filesize=11.7 K 2024-11-20T21:14:04,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/ffd2b6e61ce043f08bc4f1ce13685c6a as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/ffd2b6e61ce043f08bc4f1ce13685c6a 2024-11-20T21:14:04,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/ffd2b6e61ce043f08bc4f1ce13685c6a, entries=150, sequenceid=69, filesize=11.7 K 2024-11-20T21:14:04,512 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for f3e2abb3d075e2d6a5c2948af99b8c71 in 682ms, sequenceid=69, compaction requested=false 2024-11-20T21:14:04,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:04,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T21:14:04,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-20T21:14:04,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:04,535 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing f3e2abb3d075e2d6a5c2948af99b8c71 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T21:14:04,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=A 2024-11-20T21:14:04,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:04,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=B 2024-11-20T21:14:04,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:04,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=C 2024-11-20T21:14:04,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:04,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:04,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:04,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/9721e867538f42588695109ee4c91a56 is 50, key is test_row_0/A:col10/1732137243901/Put/seqid=0 2024-11-20T21:14:04,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:04,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:04,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137304600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:04,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137304602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:04,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741852_1028 (size=12001) 2024-11-20T21:14:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,625 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/9721e867538f42588695109ee4c91a56 2024-11-20T21:14:04,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:04,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:04,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137304619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:04,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:04,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137304622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:04,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:04,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:04,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:04,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137304623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:04,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:04,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/5e328f08584b4d2ca5a09b7297032fcc is 50, key is test_row_0/B:col10/1732137243901/Put/seqid=0 2024-11-20T21:14:04,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741853_1029 (size=12001) 2024-11-20T21:14:04,680 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/5e328f08584b4d2ca5a09b7297032fcc 2024-11-20T21:14:04,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/1179a7b2c21c49f6a64e109149d25990 is 50, key is test_row_0/C:col10/1732137243901/Put/seqid=0 2024-11-20T21:14:04,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:04,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137304729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:04,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,735 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:04,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:04,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137304738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:04,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,743 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:04,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741854_1030 (size=12001) 2024-11-20T21:14:04,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:04,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137304747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:04,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:04,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:04,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137304747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:04,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:04,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:04,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137304758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:04,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,764 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:04,800 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T21:14:04,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T21:14:04,942 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-20T21:14:04,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:04,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137304947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:04,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:04,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137304947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:04,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,950 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:04,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,951 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:04,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:04,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137304957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:04,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:04,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:04,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137304957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:04,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:04,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:04,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137304978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:04,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:04,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:05,149 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/1179a7b2c21c49f6a64e109149d25990 2024-11-20T21:14:05,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/9721e867538f42588695109ee4c91a56 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/9721e867538f42588695109ee4c91a56 2024-11-20T21:14:05,199 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/9721e867538f42588695109ee4c91a56, entries=150, sequenceid=87, filesize=11.7 K 2024-11-20T21:14:05,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/5e328f08584b4d2ca5a09b7297032fcc as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/5e328f08584b4d2ca5a09b7297032fcc 2024-11-20T21:14:05,243 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/5e328f08584b4d2ca5a09b7297032fcc, entries=150, sequenceid=87, filesize=11.7 K 2024-11-20T21:14:05,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/1179a7b2c21c49f6a64e109149d25990 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/1179a7b2c21c49f6a64e109149d25990 2024-11-20T21:14:05,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:05,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137305258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:05,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:05,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:05,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137305259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:05,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:05,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:05,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:05,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:05,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:05,279 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/1179a7b2c21c49f6a64e109149d25990, entries=150, sequenceid=87, filesize=11.7 K 2024-11-20T21:14:05,286 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for f3e2abb3d075e2d6a5c2948af99b8c71 in 751ms, sequenceid=87, compaction requested=true 2024-11-20T21:14:05,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:05,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:05,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-20T21:14:05,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-20T21:14:05,294 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-20T21:14:05,294 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0750 sec 2024-11-20T21:14:05,298 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.0860 sec 2024-11-20T21:14:05,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:05,313 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3e2abb3d075e2d6a5c2948af99b8c71 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T21:14:05,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=A 2024-11-20T21:14:05,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:05,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=B 2024-11-20T21:14:05,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:05,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=C 2024-11-20T21:14:05,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:05,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/b08c4f2dc6b54db58410ff48604a6054 is 50, key is test_row_0/A:col10/1732137244618/Put/seqid=0 2024-11-20T21:14:05,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-20T21:14:05,348 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:14:05,349 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:14:05,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-20T21:14:05,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T21:14:05,352 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:14:05,365 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:14:05,374 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:14:05,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741855_1031 (size=12001) 2024-11-20T21:14:05,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/b08c4f2dc6b54db58410ff48604a6054 2024-11-20T21:14:05,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/128b16cc6a7b443089162937fc0acfee is 50, key is test_row_0/B:col10/1732137244618/Put/seqid=0 2024-11-20T21:14:05,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:05,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137305415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:05,430 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:05,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:05,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:05,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:05,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:05,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137305425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:05,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137305430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:05,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T21:14:05,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741856_1032 (size=12001) 2024-11-20T21:14:05,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T21:14:05,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:05,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:05,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:05,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:05,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:05,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:05,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:05,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137305541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:05,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:05,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:05,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:05,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:05,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137305547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:05,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:05,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:05,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:05,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:05,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137305551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:05,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:05,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:05,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T21:14:05,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T21:14:05,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:05,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:05,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:05,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:05,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:05,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:05,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137305767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:05,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:05,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:05,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:05,774 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:05,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137305768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:05,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:05,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:05,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:05,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137305769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:05,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:05,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:05,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:05,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:05,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137305769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:05,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:05,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:05,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:05,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:05,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137305778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:05,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:05,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:05,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:05,866 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/128b16cc6a7b443089162937fc0acfee 2024-11-20T21:14:05,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T21:14:05,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:05,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:05,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:05,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:05,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:05,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/062acbfbec1348b7a9670ae189d97a6b is 50, key is test_row_0/C:col10/1732137244618/Put/seqid=0 2024-11-20T21:14:05,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741857_1033 (size=12001) 2024-11-20T21:14:05,939 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=104 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/062acbfbec1348b7a9670ae189d97a6b 2024-11-20T21:14:05,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/b08c4f2dc6b54db58410ff48604a6054 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/b08c4f2dc6b54db58410ff48604a6054 2024-11-20T21:14:05,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/b08c4f2dc6b54db58410ff48604a6054, entries=150, sequenceid=104, filesize=11.7 K 2024-11-20T21:14:05,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/128b16cc6a7b443089162937fc0acfee as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/128b16cc6a7b443089162937fc0acfee 2024-11-20T21:14:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T21:14:05,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/128b16cc6a7b443089162937fc0acfee, entries=150, sequenceid=104, filesize=11.7 K 2024-11-20T21:14:05,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/062acbfbec1348b7a9670ae189d97a6b as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/062acbfbec1348b7a9670ae189d97a6b 2024-11-20T21:14:05,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/062acbfbec1348b7a9670ae189d97a6b, entries=150, sequenceid=104, filesize=11.7 K 2024-11-20T21:14:06,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for f3e2abb3d075e2d6a5c2948af99b8c71 in 692ms, sequenceid=104, compaction requested=true 2024-11-20T21:14:06,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:06,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3e2abb3d075e2d6a5c2948af99b8c71:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:14:06,005 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:14:06,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:06,005 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:14:06,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3e2abb3d075e2d6a5c2948af99b8c71:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:14:06,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:06,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3e2abb3d075e2d6a5c2948af99b8c71:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:14:06,006 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:06,008 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:14:06,008 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): f3e2abb3d075e2d6a5c2948af99b8c71/A is initiating minor compaction (all files) 2024-11-20T21:14:06,008 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3e2abb3d075e2d6a5c2948af99b8c71/A in TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:06,008 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/b5e5077ca22a4974b765f82fcb07c3e4, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/b890b621823642d2bf0a83a6cc8ffd18, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/9721e867538f42588695109ee4c91a56, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/b08c4f2dc6b54db58410ff48604a6054] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp, totalSize=47.0 K 2024-11-20T21:14:06,008 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:14:06,008 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): f3e2abb3d075e2d6a5c2948af99b8c71/B is initiating minor compaction (all files) 2024-11-20T21:14:06,009 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3e2abb3d075e2d6a5c2948af99b8c71/B in TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:06,009 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/a2e6ac5c2ca74354bf47eb626718245d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/90d105a5f2d043a1b9646ed5c5c6e27d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/5e328f08584b4d2ca5a09b7297032fcc, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/128b16cc6a7b443089162937fc0acfee] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp, totalSize=47.0 K 2024-11-20T21:14:06,009 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting b5e5077ca22a4974b765f82fcb07c3e4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732137243214 2024-11-20T21:14:06,010 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting a2e6ac5c2ca74354bf47eb626718245d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732137243214 2024-11-20T21:14:06,010 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting b890b621823642d2bf0a83a6cc8ffd18, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732137243450 2024-11-20T21:14:06,010 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 90d105a5f2d043a1b9646ed5c5c6e27d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732137243450 2024-11-20T21:14:06,012 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e328f08584b4d2ca5a09b7297032fcc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732137243835 2024-11-20T21:14:06,012 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 128b16cc6a7b443089162937fc0acfee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732137244618 2024-11-20T21:14:06,017 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9721e867538f42588695109ee4c91a56, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732137243835 2024-11-20T21:14:06,019 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting b08c4f2dc6b54db58410ff48604a6054, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732137244618 2024-11-20T21:14:06,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-20T21:14:06,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:06,033 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing f3e2abb3d075e2d6a5c2948af99b8c71 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-20T21:14:06,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=A 2024-11-20T21:14:06,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:06,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=B 2024-11-20T21:14:06,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:06,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=C 2024-11-20T21:14:06,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:06,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/faba36ff60c54630bd5e2e9deb4323f0 is 50, key is test_row_0/A:col10/1732137245421/Put/seqid=0 2024-11-20T21:14:06,060 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3e2abb3d075e2d6a5c2948af99b8c71#A#compaction#22 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:06,061 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/bfa49b73e1874c3a8ea30cfd2050bd75 is 50, key is test_row_0/A:col10/1732137244618/Put/seqid=0 2024-11-20T21:14:06,078 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3e2abb3d075e2d6a5c2948af99b8c71#B#compaction#23 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:06,079 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/6c4ffe3b88624b4aba5c2592aa696245 is 50, key is test_row_0/B:col10/1732137244618/Put/seqid=0 2024-11-20T21:14:06,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741858_1034 (size=12001) 2024-11-20T21:14:06,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741859_1035 (size=12241) 2024-11-20T21:14:06,137 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/faba36ff60c54630bd5e2e9deb4323f0 2024-11-20T21:14:06,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:06,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:06,159 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/bfa49b73e1874c3a8ea30cfd2050bd75 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/bfa49b73e1874c3a8ea30cfd2050bd75 2024-11-20T21:14:06,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/1048ff9dcb2b4bbaafd078435c3b6c59 is 50, key is test_row_0/B:col10/1732137245421/Put/seqid=0 2024-11-20T21:14:06,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741860_1036 (size=12241) 2024-11-20T21:14:06,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741861_1037 (size=12001) 2024-11-20T21:14:06,221 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/1048ff9dcb2b4bbaafd078435c3b6c59 2024-11-20T21:14:06,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/87e6fe77c77b4affbcd3bd0d03110042 is 50, key is test_row_0/C:col10/1732137245421/Put/seqid=0 2024-11-20T21:14:06,253 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-20T21:14:06,254 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-20T21:14:06,258 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T21:14:06,258 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-20T21:14:06,259 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-20T21:14:06,259 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-20T21:14:06,260 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-20T21:14:06,260 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-20T21:14:06,267 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f3e2abb3d075e2d6a5c2948af99b8c71/A of f3e2abb3d075e2d6a5c2948af99b8c71 into bfa49b73e1874c3a8ea30cfd2050bd75(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:06,267 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:06,267 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., storeName=f3e2abb3d075e2d6a5c2948af99b8c71/A, priority=12, startTime=1732137246005; duration=0sec 2024-11-20T21:14:06,267 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:06,267 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3e2abb3d075e2d6a5c2948af99b8c71:A 2024-11-20T21:14:06,268 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:14:06,279 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:14:06,279 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): f3e2abb3d075e2d6a5c2948af99b8c71/C is initiating minor compaction (all files) 2024-11-20T21:14:06,280 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3e2abb3d075e2d6a5c2948af99b8c71/C in TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:06,280 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/0320de669d494778b6333c6e5e95bb7d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/ffd2b6e61ce043f08bc4f1ce13685c6a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/1179a7b2c21c49f6a64e109149d25990, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/062acbfbec1348b7a9670ae189d97a6b] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp, totalSize=47.0 K 2024-11-20T21:14:06,281 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0320de669d494778b6333c6e5e95bb7d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732137243214 2024-11-20T21:14:06,285 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffd2b6e61ce043f08bc4f1ce13685c6a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1732137243450 2024-11-20T21:14:06,286 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1179a7b2c21c49f6a64e109149d25990, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732137243835 2024-11-20T21:14:06,287 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 062acbfbec1348b7a9670ae189d97a6b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732137244618 2024-11-20T21:14:06,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741862_1038 (size=12001) 2024-11-20T21:14:06,303 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/87e6fe77c77b4affbcd3bd0d03110042 2024-11-20T21:14:06,307 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3e2abb3d075e2d6a5c2948af99b8c71#C#compaction#26 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:06,309 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/49f1588be7574800b59ffd075b9824fa is 50, key is test_row_0/C:col10/1732137244618/Put/seqid=0 2024-11-20T21:14:06,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/faba36ff60c54630bd5e2e9deb4323f0 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/faba36ff60c54630bd5e2e9deb4323f0 2024-11-20T21:14:06,323 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/faba36ff60c54630bd5e2e9deb4323f0, entries=150, sequenceid=119, filesize=11.7 K 2024-11-20T21:14:06,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/1048ff9dcb2b4bbaafd078435c3b6c59 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/1048ff9dcb2b4bbaafd078435c3b6c59 2024-11-20T21:14:06,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:06,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137306312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:06,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,329 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:06,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:06,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137306315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:06,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,331 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:06,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:06,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137306319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:06,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:06,345 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/1048ff9dcb2b4bbaafd078435c3b6c59, entries=150, sequenceid=119, filesize=11.7 K 2024-11-20T21:14:06,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/87e6fe77c77b4affbcd3bd0d03110042 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/87e6fe77c77b4affbcd3bd0d03110042 2024-11-20T21:14:06,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741863_1039 (size=12241) 2024-11-20T21:14:06,359 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/49f1588be7574800b59ffd075b9824fa as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/49f1588be7574800b59ffd075b9824fa 2024-11-20T21:14:06,367 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/87e6fe77c77b4affbcd3bd0d03110042, entries=150, sequenceid=119, filesize=11.7 K 2024-11-20T21:14:06,370 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for f3e2abb3d075e2d6a5c2948af99b8c71 in 337ms, sequenceid=119, compaction requested=false 2024-11-20T21:14:06,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:06,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:06,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-20T21:14:06,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-20T21:14:06,373 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f3e2abb3d075e2d6a5c2948af99b8c71/C of f3e2abb3d075e2d6a5c2948af99b8c71 into 49f1588be7574800b59ffd075b9824fa(size=12.0 K), total size for store is 23.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:06,373 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:06,373 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., storeName=f3e2abb3d075e2d6a5c2948af99b8c71/C, priority=12, startTime=1732137246006; duration=0sec 2024-11-20T21:14:06,373 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:06,373 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3e2abb3d075e2d6a5c2948af99b8c71:C 2024-11-20T21:14:06,378 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-20T21:14:06,378 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 999 msec 2024-11-20T21:14:06,386 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.0310 sec 2024-11-20T21:14:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:06,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3e2abb3d075e2d6a5c2948af99b8c71 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-20T21:14:06,444 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=A 2024-11-20T21:14:06,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:06,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=B 2024-11-20T21:14:06,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:06,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=C 2024-11-20T21:14:06,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:06,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/126c46818dc7429fbba3563004f312c7 is 50, key is test_row_0/A:col10/1732137246440/Put/seqid=0 2024-11-20T21:14:06,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-20T21:14:06,488 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:14:06,488 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:14:06,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741864_1040 (size=12151) 2024-11-20T21:14:06,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-20T21:14:06,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T21:14:06,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/126c46818dc7429fbba3563004f312c7 2024-11-20T21:14:06,493 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:14:06,494 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:14:06,494 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:14:06,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:06,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137306483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:06,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:06,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:06,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137306493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:06,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:06,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:06,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137306494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:06,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:06,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/e4accd85f5714eb587ff612fb415df1c is 50, key is test_row_0/B:col10/1732137246440/Put/seqid=0 2024-11-20T21:14:06,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741865_1041 (size=12151) 2024-11-20T21:14:06,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/e4accd85f5714eb587ff612fb415df1c 2024-11-20T21:14:06,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/2999dfbb933443bebb1121dabbe9b755 is 50, key is test_row_0/C:col10/1732137246440/Put/seqid=0 2024-11-20T21:14:06,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741866_1042 (size=12151) 2024-11-20T21:14:06,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/2999dfbb933443bebb1121dabbe9b755 2024-11-20T21:14:06,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/126c46818dc7429fbba3563004f312c7 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/126c46818dc7429fbba3563004f312c7 2024-11-20T21:14:06,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/126c46818dc7429fbba3563004f312c7, entries=150, sequenceid=139, filesize=11.9 K 2024-11-20T21:14:06,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/e4accd85f5714eb587ff612fb415df1c as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/e4accd85f5714eb587ff612fb415df1c 2024-11-20T21:14:06,594 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/6c4ffe3b88624b4aba5c2592aa696245 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/6c4ffe3b88624b4aba5c2592aa696245 2024-11-20T21:14:06,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T21:14:06,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/e4accd85f5714eb587ff612fb415df1c, entries=150, sequenceid=139, filesize=11.9 K 2024-11-20T21:14:06,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/2999dfbb933443bebb1121dabbe9b755 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/2999dfbb933443bebb1121dabbe9b755 2024-11-20T21:14:06,612 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f3e2abb3d075e2d6a5c2948af99b8c71/B of f3e2abb3d075e2d6a5c2948af99b8c71 into 6c4ffe3b88624b4aba5c2592aa696245(size=12.0 K), total size for store is 35.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:06,612 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:06,612 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., storeName=f3e2abb3d075e2d6a5c2948af99b8c71/B, priority=12, startTime=1732137246005; duration=0sec 2024-11-20T21:14:06,612 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:06,612 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3e2abb3d075e2d6a5c2948af99b8c71:B 2024-11-20T21:14:06,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:06,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137306608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:06,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137306608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:06,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:06,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:06,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137306608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:06,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,617 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:06,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/2999dfbb933443bebb1121dabbe9b755, entries=150, sequenceid=139, filesize=11.9 K 2024-11-20T21:14:06,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for f3e2abb3d075e2d6a5c2948af99b8c71 in 178ms, sequenceid=139, compaction requested=true 2024-11-20T21:14:06,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:06,623 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:06,623 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3e2abb3d075e2d6a5c2948af99b8c71:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:14:06,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:06,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3e2abb3d075e2d6a5c2948af99b8c71:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:14:06,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:06,624 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:06,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3e2abb3d075e2d6a5c2948af99b8c71:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:14:06,624 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:06,625 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:06,625 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): f3e2abb3d075e2d6a5c2948af99b8c71/A is initiating minor compaction (all files) 2024-11-20T21:14:06,625 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3e2abb3d075e2d6a5c2948af99b8c71/A in TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:06,626 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/bfa49b73e1874c3a8ea30cfd2050bd75, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/faba36ff60c54630bd5e2e9deb4323f0, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/126c46818dc7429fbba3563004f312c7] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp, totalSize=35.5 K 2024-11-20T21:14:06,627 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:06,627 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): f3e2abb3d075e2d6a5c2948af99b8c71/B is initiating minor compaction (all files) 2024-11-20T21:14:06,627 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3e2abb3d075e2d6a5c2948af99b8c71/B in TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:06,627 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfa49b73e1874c3a8ea30cfd2050bd75, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732137244618 2024-11-20T21:14:06,627 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/6c4ffe3b88624b4aba5c2592aa696245, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/1048ff9dcb2b4bbaafd078435c3b6c59, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/e4accd85f5714eb587ff612fb415df1c] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp, totalSize=35.5 K 2024-11-20T21:14:06,628 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting faba36ff60c54630bd5e2e9deb4323f0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732137245397 2024-11-20T21:14:06,628 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c4ffe3b88624b4aba5c2592aa696245, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732137244618 2024-11-20T21:14:06,629 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 1048ff9dcb2b4bbaafd078435c3b6c59, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732137245397 2024-11-20T21:14:06,629 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 126c46818dc7429fbba3563004f312c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732137246210 2024-11-20T21:14:06,630 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting e4accd85f5714eb587ff612fb415df1c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732137246210 2024-11-20T21:14:06,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-20T21:14:06,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:06,649 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing f3e2abb3d075e2d6a5c2948af99b8c71 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-20T21:14:06,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=A 2024-11-20T21:14:06,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:06,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=B 2024-11-20T21:14:06,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:06,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=C 2024-11-20T21:14:06,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:06,659 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3e2abb3d075e2d6a5c2948af99b8c71#A#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:06,659 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3e2abb3d075e2d6a5c2948af99b8c71#B#compaction#31 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:06,660 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/ed64ac5ca4714faa9d7289544b413a44 is 50, key is test_row_0/A:col10/1732137246440/Put/seqid=0 2024-11-20T21:14:06,660 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/0d533027012e4fad813b27442832038b is 50, key is test_row_0/B:col10/1732137246440/Put/seqid=0 2024-11-20T21:14:06,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/5c1561a7eadc4df0a07fb29e1c2e0b01 is 50, key is test_row_0/A:col10/1732137246487/Put/seqid=0 2024-11-20T21:14:06,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741867_1043 (size=12493) 2024-11-20T21:14:06,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741868_1044 (size=12151) 2024-11-20T21:14:06,688 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/5c1561a7eadc4df0a07fb29e1c2e0b01 2024-11-20T21:14:06,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/d9c2b1b13f50484695194beb69f9b69c is 50, key is test_row_0/B:col10/1732137246487/Put/seqid=0 2024-11-20T21:14:06,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741869_1045 (size=12493) 2024-11-20T21:14:06,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741870_1046 (size=12151) 2024-11-20T21:14:06,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:06,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T21:14:06,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:06,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137306846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:06,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:06,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:06,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137306846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:06,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:06,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:06,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137306847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:06,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:06,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,856 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:06,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137306852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:06,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:06,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:06,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137306861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:06,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:06,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:06,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137306957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:06,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:06,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:06,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137306958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:06,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:06,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:06,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137306967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:06,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:06,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:06,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137306968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:06,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:06,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137306987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:06,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:06,988 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:07,097 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/ed64ac5ca4714faa9d7289544b413a44 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/ed64ac5ca4714faa9d7289544b413a44 2024-11-20T21:14:07,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T21:14:07,129 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3e2abb3d075e2d6a5c2948af99b8c71/A of f3e2abb3d075e2d6a5c2948af99b8c71 into ed64ac5ca4714faa9d7289544b413a44(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:07,129 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:07,129 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., storeName=f3e2abb3d075e2d6a5c2948af99b8c71/A, priority=13, startTime=1732137246623; duration=0sec 2024-11-20T21:14:07,130 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:07,130 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3e2abb3d075e2d6a5c2948af99b8c71:A 2024-11-20T21:14:07,130 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:07,139 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:07,139 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): f3e2abb3d075e2d6a5c2948af99b8c71/C is initiating minor compaction (all files) 2024-11-20T21:14:07,140 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3e2abb3d075e2d6a5c2948af99b8c71/C in TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:07,140 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/49f1588be7574800b59ffd075b9824fa, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/87e6fe77c77b4affbcd3bd0d03110042, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/2999dfbb933443bebb1121dabbe9b755] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp, totalSize=35.5 K 2024-11-20T21:14:07,187 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/d9c2b1b13f50484695194beb69f9b69c 2024-11-20T21:14:07,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:07,187 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49f1588be7574800b59ffd075b9824fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=104, earliestPutTs=1732137244618 2024-11-20T21:14:07,190 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87e6fe77c77b4affbcd3bd0d03110042, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1732137245397 2024-11-20T21:14:07,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137307186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:07,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:07,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:07,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:07,193 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2999dfbb933443bebb1121dabbe9b755, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732137246210 2024-11-20T21:14:07,194 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/0d533027012e4fad813b27442832038b as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/0d533027012e4fad813b27442832038b 2024-11-20T21:14:07,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:07,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137307188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137307190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:07,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:07,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:07,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:07,197 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:07,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:07,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:07,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:07,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137307190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:07,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:07,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:07,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:07,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137307197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:07,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:07,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/aefa9772995248aca4ebb688cfb04002 is 50, key is test_row_0/C:col10/1732137246487/Put/seqid=0 2024-11-20T21:14:07,213 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3e2abb3d075e2d6a5c2948af99b8c71/B of f3e2abb3d075e2d6a5c2948af99b8c71 into 0d533027012e4fad813b27442832038b(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:07,213 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:07,213 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., storeName=f3e2abb3d075e2d6a5c2948af99b8c71/B, priority=13, startTime=1732137246624; duration=0sec 2024-11-20T21:14:07,214 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:07,214 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3e2abb3d075e2d6a5c2948af99b8c71:B 2024-11-20T21:14:07,214 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3e2abb3d075e2d6a5c2948af99b8c71#C#compaction#35 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:07,215 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/17ef6a74624746678088a1ce033b0352 is 50, key is test_row_0/C:col10/1732137246440/Put/seqid=0 2024-11-20T21:14:07,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741871_1047 (size=12151) 2024-11-20T21:14:07,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741872_1048 (size=12493) 2024-11-20T21:14:07,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:07,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137307500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:07,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:07,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:07,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:07,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:07,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137307507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:07,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:07,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:07,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:07,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:07,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137307508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:07,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:07,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:07,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:07,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:07,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137307508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:07,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:07,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137307508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:07,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:07,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:07,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:07,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:07,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:07,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:07,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T21:14:07,647 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/aefa9772995248aca4ebb688cfb04002 2024-11-20T21:14:07,679 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/17ef6a74624746678088a1ce033b0352 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/17ef6a74624746678088a1ce033b0352 2024-11-20T21:14:07,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/5c1561a7eadc4df0a07fb29e1c2e0b01 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/5c1561a7eadc4df0a07fb29e1c2e0b01 2024-11-20T21:14:07,719 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/5c1561a7eadc4df0a07fb29e1c2e0b01, entries=150, sequenceid=155, filesize=11.9 K 2024-11-20T21:14:07,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/d9c2b1b13f50484695194beb69f9b69c as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/d9c2b1b13f50484695194beb69f9b69c 2024-11-20T21:14:07,733 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/d9c2b1b13f50484695194beb69f9b69c, entries=150, sequenceid=155, filesize=11.9 K 2024-11-20T21:14:07,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/aefa9772995248aca4ebb688cfb04002 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/aefa9772995248aca4ebb688cfb04002 2024-11-20T21:14:07,741 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3e2abb3d075e2d6a5c2948af99b8c71/C of f3e2abb3d075e2d6a5c2948af99b8c71 into 17ef6a74624746678088a1ce033b0352(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:07,741 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:07,741 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., storeName=f3e2abb3d075e2d6a5c2948af99b8c71/C, priority=13, startTime=1732137246624; duration=0sec 2024-11-20T21:14:07,741 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:07,741 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3e2abb3d075e2d6a5c2948af99b8c71:C 2024-11-20T21:14:07,763 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/aefa9772995248aca4ebb688cfb04002, entries=150, sequenceid=155, filesize=11.9 K 2024-11-20T21:14:07,765 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for f3e2abb3d075e2d6a5c2948af99b8c71 in 1116ms, sequenceid=155, compaction requested=false 2024-11-20T21:14:07,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:07,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:07,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-20T21:14:07,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-20T21:14:07,777 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-20T21:14:07,777 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2790 sec 2024-11-20T21:14:07,780 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.2890 sec 2024-11-20T21:14:08,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3e2abb3d075e2d6a5c2948af99b8c71 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-20T21:14:08,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=A 2024-11-20T21:14:08,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:08,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=B 2024-11-20T21:14:08,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:08,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=C 2024-11-20T21:14:08,022 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:08,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:08,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/971af64ed749470b90164f46ce7b1bc1 is 50, key is test_row_0/A:col10/1732137248018/Put/seqid=0 2024-11-20T21:14:08,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741873_1049 (size=14541) 2024-11-20T21:14:08,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:08,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137308051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:08,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:08,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137308051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:08,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:08,067 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:08,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:08,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137308060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:08,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:08,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:08,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137308060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:08,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:08,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:08,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137308062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:08,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:08,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:08,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137308178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:08,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:08,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:08,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137308179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:08,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:08,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:08,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137308179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:08,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:08,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137308179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:08,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:08,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137308179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:08,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:08,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:08,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,184 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:08,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:08,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137308388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:08,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:08,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137308388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:08,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:08,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:08,394 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137308388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:08,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:08,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137308388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:08,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:08,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:08,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137308389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:08,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:08,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/971af64ed749470b90164f46ce7b1bc1 2024-11-20T21:14:08,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/a3e50a943b1a4cdf8cf5976f9f89018d is 50, key is test_row_0/B:col10/1732137248018/Put/seqid=0 2024-11-20T21:14:08,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741874_1050 (size=12151) 2024-11-20T21:14:08,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-20T21:14:08,647 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:14:08,648 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:14:08,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-20T21:14:08,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T21:14:08,654 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:14:08,657 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:14:08,657 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:14:08,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:08,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137308698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:08,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:08,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137308700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:08,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:08,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:08,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:08,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137308707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:08,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:08,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137308708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:08,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:08,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:08,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137308708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:08,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:08,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:08,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T21:14:08,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T21:14:08,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:08,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:08,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:08,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:08,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:08,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:08,899 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/a3e50a943b1a4cdf8cf5976f9f89018d 2024-11-20T21:14:08,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/920ab2711bda41eabfa693d8171e5e11 is 50, key is test_row_0/C:col10/1732137248018/Put/seqid=0 2024-11-20T21:14:08,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741875_1051 (size=12151) 2024-11-20T21:14:08,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/920ab2711bda41eabfa693d8171e5e11 2024-11-20T21:14:08,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/971af64ed749470b90164f46ce7b1bc1 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/971af64ed749470b90164f46ce7b1bc1 2024-11-20T21:14:08,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/971af64ed749470b90164f46ce7b1bc1, entries=200, sequenceid=177, filesize=14.2 K 2024-11-20T21:14:08,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/a3e50a943b1a4cdf8cf5976f9f89018d as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/a3e50a943b1a4cdf8cf5976f9f89018d 2024-11-20T21:14:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T21:14:08,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/a3e50a943b1a4cdf8cf5976f9f89018d, entries=150, sequenceid=177, filesize=11.9 K 2024-11-20T21:14:08,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/920ab2711bda41eabfa693d8171e5e11 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/920ab2711bda41eabfa693d8171e5e11 2024-11-20T21:14:08,980 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/920ab2711bda41eabfa693d8171e5e11, entries=150, sequenceid=177, filesize=11.9 K 2024-11-20T21:14:08,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for f3e2abb3d075e2d6a5c2948af99b8c71 in 961ms, sequenceid=177, compaction requested=true 2024-11-20T21:14:08,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:08,982 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:08,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3e2abb3d075e2d6a5c2948af99b8c71:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:14:08,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:08,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3e2abb3d075e2d6a5c2948af99b8c71:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:14:08,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:08,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3e2abb3d075e2d6a5c2948af99b8c71:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:14:08,983 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:08,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:08,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-20T21:14:08,985 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:08,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:08,985 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): f3e2abb3d075e2d6a5c2948af99b8c71/A is initiating minor compaction (all files) 2024-11-20T21:14:08,985 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing f3e2abb3d075e2d6a5c2948af99b8c71 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-20T21:14:08,985 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3e2abb3d075e2d6a5c2948af99b8c71/A in TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:08,985 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/ed64ac5ca4714faa9d7289544b413a44, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/5c1561a7eadc4df0a07fb29e1c2e0b01, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/971af64ed749470b90164f46ce7b1bc1] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp, totalSize=38.3 K 2024-11-20T21:14:08,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=A 2024-11-20T21:14:08,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:08,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=B 2024-11-20T21:14:08,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:08,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=C 2024-11-20T21:14:08,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:08,986 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed64ac5ca4714faa9d7289544b413a44, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732137246210 2024-11-20T21:14:08,986 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:08,986 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): f3e2abb3d075e2d6a5c2948af99b8c71/B is initiating minor compaction (all files) 2024-11-20T21:14:08,987 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3e2abb3d075e2d6a5c2948af99b8c71/B in TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:08,987 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/0d533027012e4fad813b27442832038b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/d9c2b1b13f50484695194beb69f9b69c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/a3e50a943b1a4cdf8cf5976f9f89018d] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp, totalSize=35.9 K 2024-11-20T21:14:08,987 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c1561a7eadc4df0a07fb29e1c2e0b01, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732137246470 2024-11-20T21:14:08,987 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d533027012e4fad813b27442832038b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732137246210 2024-11-20T21:14:08,988 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 971af64ed749470b90164f46ce7b1bc1, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732137246842 2024-11-20T21:14:08,988 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting d9c2b1b13f50484695194beb69f9b69c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732137246470 2024-11-20T21:14:08,989 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting a3e50a943b1a4cdf8cf5976f9f89018d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732137246842 2024-11-20T21:14:09,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/68ccace2bd3949208c2a4717809293ff is 50, key is test_row_0/A:col10/1732137248057/Put/seqid=0 2024-11-20T21:14:09,015 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3e2abb3d075e2d6a5c2948af99b8c71#A#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:09,016 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/e292dbdc37614c298748c1c41f55b9cf is 50, key is test_row_0/A:col10/1732137248018/Put/seqid=0 2024-11-20T21:14:09,025 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3e2abb3d075e2d6a5c2948af99b8c71#B#compaction#41 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:09,026 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/c363e517351f4218a59ada1fa15bd104 is 50, key is test_row_0/B:col10/1732137248018/Put/seqid=0 2024-11-20T21:14:09,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741878_1054 (size=12595) 2024-11-20T21:14:09,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741876_1052 (size=12151) 2024-11-20T21:14:09,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741877_1053 (size=12595) 2024-11-20T21:14:09,111 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/e292dbdc37614c298748c1c41f55b9cf as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/e292dbdc37614c298748c1c41f55b9cf 2024-11-20T21:14:09,121 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3e2abb3d075e2d6a5c2948af99b8c71/A of f3e2abb3d075e2d6a5c2948af99b8c71 into e292dbdc37614c298748c1c41f55b9cf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:09,121 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:09,121 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., storeName=f3e2abb3d075e2d6a5c2948af99b8c71/A, priority=13, startTime=1732137248982; duration=0sec 2024-11-20T21:14:09,121 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:09,122 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3e2abb3d075e2d6a5c2948af99b8c71:A 2024-11-20T21:14:09,122 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:09,123 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:09,123 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): f3e2abb3d075e2d6a5c2948af99b8c71/C is initiating minor compaction (all files) 2024-11-20T21:14:09,123 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3e2abb3d075e2d6a5c2948af99b8c71/C in TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:09,123 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/17ef6a74624746678088a1ce033b0352, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/aefa9772995248aca4ebb688cfb04002, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/920ab2711bda41eabfa693d8171e5e11] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp, totalSize=35.9 K 2024-11-20T21:14:09,124 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17ef6a74624746678088a1ce033b0352, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1732137246210 2024-11-20T21:14:09,125 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting aefa9772995248aca4ebb688cfb04002, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732137246470 2024-11-20T21:14:09,125 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 920ab2711bda41eabfa693d8171e5e11, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732137246842 2024-11-20T21:14:09,144 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3e2abb3d075e2d6a5c2948af99b8c71#C#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:09,145 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/df92c60a5ec94b29b886d7a7415cff4a is 50, key is test_row_0/C:col10/1732137248018/Put/seqid=0 2024-11-20T21:14:09,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741879_1055 (size=12595) 2024-11-20T21:14:09,189 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/df92c60a5ec94b29b886d7a7415cff4a as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/df92c60a5ec94b29b886d7a7415cff4a 2024-11-20T21:14:09,199 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3e2abb3d075e2d6a5c2948af99b8c71/C of f3e2abb3d075e2d6a5c2948af99b8c71 into df92c60a5ec94b29b886d7a7415cff4a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:09,199 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:09,199 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., storeName=f3e2abb3d075e2d6a5c2948af99b8c71/C, priority=13, startTime=1732137248983; duration=0sec 2024-11-20T21:14:09,199 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:09,199 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3e2abb3d075e2d6a5c2948af99b8c71:C 2024-11-20T21:14:09,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:09,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:09,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137309244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:09,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:09,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137309244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:09,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:09,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:09,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137309246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:09,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137309252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:09,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:09,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137309254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:09,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:09,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:09,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:09,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T21:14:09,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:09,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137309368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:09,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137309369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:09,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:09,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137309367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137309369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:09,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,373 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:09,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137309369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:09,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:09,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:09,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,379 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:09,460 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/c363e517351f4218a59ada1fa15bd104 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/c363e517351f4218a59ada1fa15bd104 2024-11-20T21:14:09,470 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3e2abb3d075e2d6a5c2948af99b8c71/B of f3e2abb3d075e2d6a5c2948af99b8c71 into c363e517351f4218a59ada1fa15bd104(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:09,470 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:09,470 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., storeName=f3e2abb3d075e2d6a5c2948af99b8c71/B, priority=13, startTime=1732137248983; duration=0sec 2024-11-20T21:14:09,471 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:09,471 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3e2abb3d075e2d6a5c2948af99b8c71:B 2024-11-20T21:14:09,489 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/68ccace2bd3949208c2a4717809293ff 2024-11-20T21:14:09,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/0803397ccc4f43ccb64789985598e618 is 50, key is test_row_0/B:col10/1732137248057/Put/seqid=0 2024-11-20T21:14:09,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741880_1056 (size=12151) 2024-11-20T21:14:09,572 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/0803397ccc4f43ccb64789985598e618 2024-11-20T21:14:09,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:09,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137309578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:09,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:09,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:09,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137309581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:09,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:09,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:09,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137309582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:09,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:09,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:09,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137309593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:09,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:09,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137309593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:09,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:09,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:09,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/1e4b397403084f5e87630f51a06a1beb is 50, key is test_row_0/C:col10/1732137248057/Put/seqid=0 2024-11-20T21:14:09,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741881_1057 (size=12151) 2024-11-20T21:14:09,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T21:14:09,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:09,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137309887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:09,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,895 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:09,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:09,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137309897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:09,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:09,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:09,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137309897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:09,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:09,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:09,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137309908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:09,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:09,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137309909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:09,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:09,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:09,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:10,047 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=190 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/1e4b397403084f5e87630f51a06a1beb 2024-11-20T21:14:10,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/68ccace2bd3949208c2a4717809293ff as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/68ccace2bd3949208c2a4717809293ff 2024-11-20T21:14:10,071 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/68ccace2bd3949208c2a4717809293ff, entries=150, sequenceid=190, filesize=11.9 K 2024-11-20T21:14:10,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/0803397ccc4f43ccb64789985598e618 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/0803397ccc4f43ccb64789985598e618 2024-11-20T21:14:10,080 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/0803397ccc4f43ccb64789985598e618, entries=150, sequenceid=190, filesize=11.9 K 2024-11-20T21:14:10,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/1e4b397403084f5e87630f51a06a1beb as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/1e4b397403084f5e87630f51a06a1beb 2024-11-20T21:14:10,102 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/1e4b397403084f5e87630f51a06a1beb, entries=150, sequenceid=190, filesize=11.9 K 2024-11-20T21:14:10,104 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f3e2abb3d075e2d6a5c2948af99b8c71 in 1119ms, sequenceid=190, compaction requested=false 2024-11-20T21:14:10,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:10,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:10,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-20T21:14:10,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-20T21:14:10,120 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-20T21:14:10,120 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4560 sec 2024-11-20T21:14:10,123 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.4730 sec 2024-11-20T21:14:10,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:10,414 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3e2abb3d075e2d6a5c2948af99b8c71 3/3 column families, dataSize=114.05 KB heapSize=339.66 KB 2024-11-20T21:14:10,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=A 2024-11-20T21:14:10,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732137249249, max timestamp=1732137250412Num uniques 859; 2024-11-20T21:14:10,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732137249249, max timestamp=1732137250412Num uniques 859; 2024-11-20T21:14:10,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T21:14:10,433 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T21:14:10,425 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1732137249222, max timestamp=1732137250412Num uniques 859; 2024-11-20T21:14:10,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=8286, suffix heap size=117064,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=181 2024-11-20T21:14:10,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:14:10,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=B 2024-11-20T21:14:10,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:14:10,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=C 2024-11-20T21:14:10,441 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:14:10,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/5e747b9214544baf8bf29445ae449995 is 50, key is test_row_0/A:col10/1732137249249/Put/seqid=0 2024-11-20T21:14:10,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741882_1058 (size=12151) 2024-11-20T21:14:10,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.93 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/5e747b9214544baf8bf29445ae449995 2024-11-20T21:14:10,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/e43b83647ea3436c99d884b2ebdb06b4 is 50, key is test_row_0/B:col10/1732137249249/Put/seqid=0 2024-11-20T21:14:10,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732137250488, max timestamp=1732137250497Num uniques 859; 2024-11-20T21:14:10,505 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=8102, suffix heap size=117064,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=177 2024-11-20T21:14:10,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732137250488, max timestamp=1732137250497Num uniques 859; 2024-11-20T21:14:10,508 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T21:14:10,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732137250488, max timestamp=1732137250497Num uniques 859; 2024-11-20T21:14:10,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-20T21:14:10,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741883_1059 (size=12151) 2024-11-20T21:14:10,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/e43b83647ea3436c99d884b2ebdb06b4 2024-11-20T21:14:10,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/45597aca93e94ff28dc49f37cfa89a7e is 50, key is test_row_0/C:col10/1732137249249/Put/seqid=0 2024-11-20T21:14:10,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12265, totalHeapSize=28112, min timestamp=1732137250532, max timestamp=1732137250573Num uniques 1036; 2024-11-20T21:14:10,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47443, new segment data size=12265, suffix heap size=135712,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=268 2024-11-20T21:14:10,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741884_1060 (size=12151) 2024-11-20T21:14:10,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732137250550, max timestamp=1732137250573Num uniques 1009; 2024-11-20T21:14:10,587 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-11-20T21:14:10,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732137250550, max timestamp=1732137250573Num uniques 1018; 2024-11-20T21:14:10,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-20T21:14:10,640 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1732137250604, max timestamp=1732137250633Num uniques 1009; 2024-11-20T21:14:10,641 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=14972, suffix heap size=132904,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=327 2024-11-20T21:14:10,640 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1732137250604, max timestamp=1732137250633Num uniques 1027; 2024-11-20T21:14:10,642 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=14972, suffix heap size=134776,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=327 2024-11-20T21:14:10,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1732137250604, max timestamp=1732137250633Num uniques 1127; 2024-11-20T21:14:10,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51612, new segment data size=14972, suffix heap size=145176,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1127, new segment cells count=327 2024-11-20T21:14:10,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732137250676, max timestamp=1732137250691Num uniques 1186; 2024-11-20T21:14:10,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=13096, suffix heap size=151312,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=286 2024-11-20T21:14:10,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732137250676, max timestamp=1732137250691Num uniques 1186; 2024-11-20T21:14:10,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=13096, suffix heap size=151312,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=286 2024-11-20T21:14:10,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1732137250676, max timestamp=1732137250691Num uniques 1186; 2024-11-20T21:14:10,700 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=13096, suffix heap size=151312,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=286 2024-11-20T21:14:10,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-20T21:14:10,808 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:14:10,808 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:14:10,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-20T21:14:10,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:10,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T21:14:10,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137310798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:10,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:10,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:10,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:10,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:10,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18090, totalHeapSize=41320, min timestamp=1732137250761, max timestamp=1732137250804Num uniques 1145; 2024-11-20T21:14:10,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=18090, suffix heap size=147048,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=395 2024-11-20T21:14:10,814 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:14:10,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137310800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:10,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:10,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:10,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:10,815 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:14:10,815 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:14:10,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732137250776, max timestamp=1732137250804Num uniques 1145; 2024-11-20T21:14:10,825 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=8930, suffix heap size=147048,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=195 2024-11-20T21:14:10,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18090, totalHeapSize=41320, min timestamp=1732137250761, max timestamp=1732137250804Num uniques 1145; 2024-11-20T21:14:10,826 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=18090, suffix heap size=147048,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=395 2024-11-20T21:14:10,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:10,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137310892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:10,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:10,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:10,901 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:10,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1732137250858, max timestamp=1732137250897Num uniques 1254; 2024-11-20T21:14:10,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1732137250858, max timestamp=1732137250897Num uniques 1254; 2024-11-20T21:14:10,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=57430, new segment data size=7050, suffix heap size=158384,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1254, new segment cells count=154 2024-11-20T21:14:10,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=57430, new segment data size=7050, suffix heap size=158384,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1254, new segment cells count=154 2024-11-20T21:14:10,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1732137250858, max timestamp=1732137250897Num uniques 1054; 2024-11-20T21:14:10,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-20T21:14:10,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:10,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137310893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:10,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:10,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:10,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:10,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T21:14:10,969 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T21:14:10,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:10,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:10,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:10,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:10,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:10,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:10,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/45597aca93e94ff28dc49f37cfa89a7e 2024-11-20T21:14:10,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/5e747b9214544baf8bf29445ae449995 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/5e747b9214544baf8bf29445ae449995 2024-11-20T21:14:11,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9745, totalHeapSize=22392, min timestamp=1732137250980, max timestamp=1732137251000Num uniques 1013; 2024-11-20T21:14:11,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46385, new segment data size=9745, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-20T21:14:11,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732137250980, max timestamp=1732137250997Num uniques 1013; 2024-11-20T21:14:11,008 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-20T21:14:11,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/5e747b9214544baf8bf29445ae449995, entries=150, sequenceid=214, filesize=11.9 K 2024-11-20T21:14:11,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732137250980, max timestamp=1732137250997Num uniques 1013; 2024-11-20T21:14:11,013 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-20T21:14:11,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/e43b83647ea3436c99d884b2ebdb06b4 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/e43b83647ea3436c99d884b2ebdb06b4 2024-11-20T21:14:11,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/e43b83647ea3436c99d884b2ebdb06b4, entries=150, sequenceid=214, filesize=11.9 K 2024-11-20T21:14:11,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/45597aca93e94ff28dc49f37cfa89a7e as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/45597aca93e94ff28dc49f37cfa89a7e 2024-11-20T21:14:11,057 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/45597aca93e94ff28dc49f37cfa89a7e, entries=150, sequenceid=214, filesize=11.9 K 2024-11-20T21:14:11,060 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~27.82 KB/28492, heapSize ~68.42 KB/70064, currentSize=127.47 KB/130530 for f3e2abb3d075e2d6a5c2948af99b8c71 in 647ms, sequenceid=214, compaction requested=true 2024-11-20T21:14:11,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:11,060 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:11,062 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:11,062 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): f3e2abb3d075e2d6a5c2948af99b8c71/A is initiating minor compaction (all files) 2024-11-20T21:14:11,062 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3e2abb3d075e2d6a5c2948af99b8c71/A in TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:11,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:11,062 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/e292dbdc37614c298748c1c41f55b9cf, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/68ccace2bd3949208c2a4717809293ff, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/5e747b9214544baf8bf29445ae449995] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp, totalSize=36.0 K 2024-11-20T21:14:11,063 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting e292dbdc37614c298748c1c41f55b9cf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732137246842 2024-11-20T21:14:11,063 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68ccace2bd3949208c2a4717809293ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732137248033 2024-11-20T21:14:11,064 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e747b9214544baf8bf29445ae449995, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732137249249 2024-11-20T21:14:11,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=422, cellSize=19322, totalHeapSize=44128, min timestamp=1732137250991, max timestamp=1732137251063Num uniques 1072; 2024-11-20T21:14:11,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=19322, suffix heap size=139456,new segment heap size=44128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=422 2024-11-20T21:14:11,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=422, cellSize=19322, totalHeapSize=44128, min timestamp=1732137250991, max timestamp=1732137251063Num uniques 1072; 2024-11-20T21:14:11,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=422, cellSize=19322, totalHeapSize=44128, min timestamp=1732137250991, max timestamp=1732137251063Num uniques 1072; 2024-11-20T21:14:11,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=19322, suffix heap size=139456,new segment heap size=44128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=422 2024-11-20T21:14:11,068 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=19322, suffix heap size=139456,new segment heap size=44128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=422 2024-11-20T21:14:11,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3e2abb3d075e2d6a5c2948af99b8c71:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:14:11,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:11,072 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:11,074 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:11,074 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): f3e2abb3d075e2d6a5c2948af99b8c71/B is initiating minor compaction (all files) 2024-11-20T21:14:11,074 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3e2abb3d075e2d6a5c2948af99b8c71/B in TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:11,074 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/c363e517351f4218a59ada1fa15bd104, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/0803397ccc4f43ccb64789985598e618, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/e43b83647ea3436c99d884b2ebdb06b4] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp, totalSize=36.0 K 2024-11-20T21:14:11,075 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting c363e517351f4218a59ada1fa15bd104, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732137246842 2024-11-20T21:14:11,075 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 0803397ccc4f43ccb64789985598e618, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732137248033 2024-11-20T21:14:11,076 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting e43b83647ea3436c99d884b2ebdb06b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732137249249 2024-11-20T21:14:11,083 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3e2abb3d075e2d6a5c2948af99b8c71#A#compaction#48 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:11,084 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/4320625392424c16b1d4e5ddb22b6d15 is 50, key is test_row_0/A:col10/1732137249249/Put/seqid=0 2024-11-20T21:14:11,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3e2abb3d075e2d6a5c2948af99b8c71:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:14:11,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:11,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3e2abb3d075e2d6a5c2948af99b8c71:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:14:11,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:11,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3e2abb3d075e2d6a5c2948af99b8c71 3/3 column families, dataSize=100.63 KB heapSize=260.72 KB 2024-11-20T21:14:11,090 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3e2abb3d075e2d6a5c2948af99b8c71#B#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:11,091 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/a15e758efa194486807468c0f1d296eb is 50, key is test_row_0/B:col10/1732137249249/Put/seqid=0 2024-11-20T21:14:11,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=A 2024-11-20T21:14:11,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:14:11,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=B 2024-11-20T21:14:11,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:14:11,102 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=C 2024-11-20T21:14:11,103 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:14:11,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T21:14:11,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:11,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:11,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:11,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:11,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:11,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:11,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741885_1061 (size=12697) 2024-11-20T21:14:11,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T21:14:11,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/ebd858b725474d16b232ace1e4816e44 is 50, key is test_row_0/A:col10/1732137251083/Put/seqid=0 2024-11-20T21:14:11,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741886_1062 (size=12697) 2024-11-20T21:14:11,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:11,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137311135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:11,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:11,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137311138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:11,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,155 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:11,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:11,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137311140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:11,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:11,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:11,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:11,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137311143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,157 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:11,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:11,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137311154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:11,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:11,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741887_1063 (size=12301) 2024-11-20T21:14:11,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:11,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137311264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:11,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,274 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:11,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137311269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:11,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,277 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:11,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137311269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:11,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:11,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,279 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:11,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137311269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:11,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:11,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T21:14:11,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:11,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:11,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:11,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:11,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:11,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137311280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:11,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,287 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:11,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:11,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T21:14:11,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:11,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:11,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:11,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:11,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:11,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:11,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T21:14:11,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:11,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137311480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:11,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:11,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:11,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137311487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:11,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:11,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:11,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137311487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:11,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:11,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137311487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:11,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:11,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:11,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:11,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137311499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:11,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,503 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:11,571 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/a15e758efa194486807468c0f1d296eb as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/a15e758efa194486807468c0f1d296eb 2024-11-20T21:14:11,580 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/4320625392424c16b1d4e5ddb22b6d15 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/4320625392424c16b1d4e5ddb22b6d15 2024-11-20T21:14:11,598 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3e2abb3d075e2d6a5c2948af99b8c71/B of f3e2abb3d075e2d6a5c2948af99b8c71 into a15e758efa194486807468c0f1d296eb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:11,598 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:11,598 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., storeName=f3e2abb3d075e2d6a5c2948af99b8c71/B, priority=13, startTime=1732137251072; duration=0sec 2024-11-20T21:14:11,598 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:11,598 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3e2abb3d075e2d6a5c2948af99b8c71:B 2024-11-20T21:14:11,598 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:11,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/ebd858b725474d16b232ace1e4816e44 2024-11-20T21:14:11,601 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:11,601 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): f3e2abb3d075e2d6a5c2948af99b8c71/C is initiating minor compaction (all files) 2024-11-20T21:14:11,601 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3e2abb3d075e2d6a5c2948af99b8c71/C in TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:11,601 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/df92c60a5ec94b29b886d7a7415cff4a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/1e4b397403084f5e87630f51a06a1beb, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/45597aca93e94ff28dc49f37cfa89a7e] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp, totalSize=36.0 K 2024-11-20T21:14:11,602 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting df92c60a5ec94b29b886d7a7415cff4a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1732137246842 2024-11-20T21:14:11,645 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e4b397403084f5e87630f51a06a1beb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=190, earliestPutTs=1732137248033 2024-11-20T21:14:11,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T21:14:11,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:11,646 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3e2abb3d075e2d6a5c2948af99b8c71/A of f3e2abb3d075e2d6a5c2948af99b8c71 into 4320625392424c16b1d4e5ddb22b6d15(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:11,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:11,646 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:11,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:11,646 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., storeName=f3e2abb3d075e2d6a5c2948af99b8c71/A, priority=13, startTime=1732137251060; duration=0sec 2024-11-20T21:14:11,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:11,646 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:11,646 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3e2abb3d075e2d6a5c2948af99b8c71:A 2024-11-20T21:14:11,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:11,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:11,648 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 45597aca93e94ff28dc49f37cfa89a7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732137249249 2024-11-20T21:14:11,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/86d5e90fc0a7447f8ea3c95cdc4399fd is 50, key is test_row_0/B:col10/1732137251083/Put/seqid=0 2024-11-20T21:14:11,690 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3e2abb3d075e2d6a5c2948af99b8c71#C#compaction#52 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:11,691 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/f48bb882d63d4494acfcb880e9d45ce9 is 50, key is test_row_0/C:col10/1732137249249/Put/seqid=0 2024-11-20T21:14:11,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741888_1064 (size=12301) 2024-11-20T21:14:11,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/86d5e90fc0a7447f8ea3c95cdc4399fd 2024-11-20T21:14:11,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741889_1065 (size=12697) 2024-11-20T21:14:11,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/e5ad822a561d4289a4d5c93468cccf58 is 50, key is test_row_0/C:col10/1732137251083/Put/seqid=0 2024-11-20T21:14:11,783 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/f48bb882d63d4494acfcb880e9d45ce9 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/f48bb882d63d4494acfcb880e9d45ce9 2024-11-20T21:14:11,793 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3e2abb3d075e2d6a5c2948af99b8c71/C of f3e2abb3d075e2d6a5c2948af99b8c71 into f48bb882d63d4494acfcb880e9d45ce9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:11,793 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:11,793 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., storeName=f3e2abb3d075e2d6a5c2948af99b8c71/C, priority=13, startTime=1732137251089; duration=0sec 2024-11-20T21:14:11,793 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:11,794 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3e2abb3d075e2d6a5c2948af99b8c71:C 2024-11-20T21:14:11,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:11,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137311798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:11,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T21:14:11,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:11,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:11,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:11,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:11,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:11,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:11,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:11,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741890_1066 (size=12301) 2024-11-20T21:14:11,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/e5ad822a561d4289a4d5c93468cccf58 2024-11-20T21:14:11,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/ebd858b725474d16b232ace1e4816e44 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/ebd858b725474d16b232ace1e4816e44 2024-11-20T21:14:11,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/ebd858b725474d16b232ace1e4816e44, entries=150, sequenceid=362, filesize=12.0 K 2024-11-20T21:14:11,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/86d5e90fc0a7447f8ea3c95cdc4399fd as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/86d5e90fc0a7447f8ea3c95cdc4399fd 2024-11-20T21:14:11,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:11,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137311811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:11,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:11,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137311811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:11,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137311811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:11,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:11,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:11,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137311811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:11,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:11,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:11,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/86d5e90fc0a7447f8ea3c95cdc4399fd, entries=150, sequenceid=362, filesize=12.0 K 2024-11-20T21:14:11,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/e5ad822a561d4289a4d5c93468cccf58 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/e5ad822a561d4289a4d5c93468cccf58 2024-11-20T21:14:11,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/e5ad822a561d4289a4d5c93468cccf58, entries=150, sequenceid=362, filesize=12.0 K 2024-11-20T21:14:11,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~280.59 KB/287328, currentSize=80.51 KB/82440 for f3e2abb3d075e2d6a5c2948af99b8c71 in 769ms, sequenceid=362, compaction requested=false 2024-11-20T21:14:11,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:11,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-20T21:14:11,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:11,963 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing f3e2abb3d075e2d6a5c2948af99b8c71 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-20T21:14:11,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=A 2024-11-20T21:14:11,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:11,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=B 2024-11-20T21:14:11,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:11,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=C 2024-11-20T21:14:11,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:11,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T21:14:11,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/7e910931a5b94696bfe300ed036ca4e6 is 50, key is test_row_0/A:col10/1732137251139/Put/seqid=0 2024-11-20T21:14:12,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741891_1067 (size=12301) 2024-11-20T21:14:12,035 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/7e910931a5b94696bfe300ed036ca4e6 2024-11-20T21:14:12,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/f0e0cd6192db477f95fffcc87f23f067 is 50, key is test_row_0/B:col10/1732137251139/Put/seqid=0 2024-11-20T21:14:12,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741892_1068 (size=12301) 2024-11-20T21:14:12,117 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/f0e0cd6192db477f95fffcc87f23f067 2024-11-20T21:14:12,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/54dbc05acd134e1b835a63fa2fc34b20 is 50, key is test_row_0/C:col10/1732137251139/Put/seqid=0 2024-11-20T21:14:12,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741893_1069 (size=12301) 2024-11-20T21:14:12,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:12,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:12,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137312401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:12,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:12,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137312401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:12,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:12,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:12,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137312403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:12,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:12,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137312403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:12,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:12,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137312411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:12,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:12,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137312518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:12,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:12,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:12,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137312518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137312518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:12,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:12,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:12,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:12,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137312518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:12,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:12,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:12,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:12,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137312528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:12,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:12,624 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/54dbc05acd134e1b835a63fa2fc34b20 2024-11-20T21:14:12,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/7e910931a5b94696bfe300ed036ca4e6 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/7e910931a5b94696bfe300ed036ca4e6 2024-11-20T21:14:12,636 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/7e910931a5b94696bfe300ed036ca4e6, entries=150, sequenceid=380, filesize=12.0 K 2024-11-20T21:14:12,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/f0e0cd6192db477f95fffcc87f23f067 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/f0e0cd6192db477f95fffcc87f23f067 2024-11-20T21:14:12,644 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/f0e0cd6192db477f95fffcc87f23f067, entries=150, sequenceid=380, filesize=12.0 K 2024-11-20T21:14:12,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/54dbc05acd134e1b835a63fa2fc34b20 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/54dbc05acd134e1b835a63fa2fc34b20 2024-11-20T21:14:12,658 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/54dbc05acd134e1b835a63fa2fc34b20, entries=150, sequenceid=380, filesize=12.0 K 2024-11-20T21:14:12,661 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for f3e2abb3d075e2d6a5c2948af99b8c71 in 698ms, sequenceid=380, compaction requested=true 2024-11-20T21:14:12,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:12,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:12,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-20T21:14:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-20T21:14:12,669 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-20T21:14:12,670 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8490 sec 2024-11-20T21:14:12,675 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.8610 sec 2024-11-20T21:14:12,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:12,734 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3e2abb3d075e2d6a5c2948af99b8c71 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-20T21:14:12,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=A 2024-11-20T21:14:12,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:12,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=B 2024-11-20T21:14:12,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:12,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=C 2024-11-20T21:14:12,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:12,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/754ee91f2d7140158d7d976e750cd74f is 50, key is test_row_0/A:col10/1732137252409/Put/seqid=0 2024-11-20T21:14:12,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:12,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137312756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:12,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:12,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:12,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137312763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:12,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:12,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:12,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137312759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:12,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:12,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137312766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:12,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:12,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137312767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:12,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:12,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741894_1070 (size=12301) 2024-11-20T21:14:12,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/754ee91f2d7140158d7d976e750cd74f 2024-11-20T21:14:12,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/c5bdb08e67794f1e9642f35098bfe58b is 50, key is test_row_0/B:col10/1732137252409/Put/seqid=0 2024-11-20T21:14:12,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741895_1071 (size=12301) 2024-11-20T21:14:12,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/c5bdb08e67794f1e9642f35098bfe58b 2024-11-20T21:14:12,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/c624a12abf294e6f8fb00d950b9e802a is 50, key is test_row_0/C:col10/1732137252409/Put/seqid=0 2024-11-20T21:14:12,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:12,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137312877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:12,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:12,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137312877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:12,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:12,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137312877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:12,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:12,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:12,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,881 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:12,881 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:12,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137312877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:12,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:12,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137312877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:12,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:12,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:12,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:12,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741896_1072 (size=12301) 2024-11-20T21:14:12,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/c624a12abf294e6f8fb00d950b9e802a 2024-11-20T21:14:12,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/754ee91f2d7140158d7d976e750cd74f as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/754ee91f2d7140158d7d976e750cd74f 2024-11-20T21:14:12,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/754ee91f2d7140158d7d976e750cd74f, entries=150, sequenceid=400, filesize=12.0 K 2024-11-20T21:14:12,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/c5bdb08e67794f1e9642f35098bfe58b as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/c5bdb08e67794f1e9642f35098bfe58b 2024-11-20T21:14:12,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/c5bdb08e67794f1e9642f35098bfe58b, entries=150, sequenceid=400, filesize=12.0 K 2024-11-20T21:14:12,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/c624a12abf294e6f8fb00d950b9e802a as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/c624a12abf294e6f8fb00d950b9e802a 2024-11-20T21:14:12,918 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/c624a12abf294e6f8fb00d950b9e802a, entries=150, sequenceid=400, filesize=12.0 K 2024-11-20T21:14:12,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for f3e2abb3d075e2d6a5c2948af99b8c71 in 186ms, sequenceid=400, compaction requested=true 2024-11-20T21:14:12,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:12,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3e2abb3d075e2d6a5c2948af99b8c71:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:14:12,920 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:14:12,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:12,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3e2abb3d075e2d6a5c2948af99b8c71:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:14:12,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:12,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3e2abb3d075e2d6a5c2948af99b8c71:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:14:12,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T21:14:12,922 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:14:12,924 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49600 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:14:12,925 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): f3e2abb3d075e2d6a5c2948af99b8c71/A is initiating minor compaction (all files) 2024-11-20T21:14:12,925 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3e2abb3d075e2d6a5c2948af99b8c71/A in TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:12,925 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/4320625392424c16b1d4e5ddb22b6d15, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/ebd858b725474d16b232ace1e4816e44, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/7e910931a5b94696bfe300ed036ca4e6, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/754ee91f2d7140158d7d976e750cd74f] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp, totalSize=48.4 K 2024-11-20T21:14:12,926 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4320625392424c16b1d4e5ddb22b6d15, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732137249249 2024-11-20T21:14:12,927 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebd858b725474d16b232ace1e4816e44, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732137251075 2024-11-20T21:14:12,929 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49600 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:14:12,929 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e910931a5b94696bfe300ed036ca4e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732137251137 2024-11-20T21:14:12,929 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): f3e2abb3d075e2d6a5c2948af99b8c71/B is initiating minor compaction (all files) 2024-11-20T21:14:12,929 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3e2abb3d075e2d6a5c2948af99b8c71/B in TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:12,929 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/a15e758efa194486807468c0f1d296eb, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/86d5e90fc0a7447f8ea3c95cdc4399fd, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/f0e0cd6192db477f95fffcc87f23f067, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/c5bdb08e67794f1e9642f35098bfe58b] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp, totalSize=48.4 K 2024-11-20T21:14:12,929 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 754ee91f2d7140158d7d976e750cd74f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1732137252409 2024-11-20T21:14:12,930 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting a15e758efa194486807468c0f1d296eb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732137249249 2024-11-20T21:14:12,930 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 86d5e90fc0a7447f8ea3c95cdc4399fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732137251075 2024-11-20T21:14:12,931 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting f0e0cd6192db477f95fffcc87f23f067, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732137251137 2024-11-20T21:14:12,931 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting c5bdb08e67794f1e9642f35098bfe58b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1732137252409 2024-11-20T21:14:12,948 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3e2abb3d075e2d6a5c2948af99b8c71#A#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:12,949 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/9727327d25f6455f94b2e3dfa0da9fdd is 50, key is test_row_0/A:col10/1732137252409/Put/seqid=0 2024-11-20T21:14:12,959 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3e2abb3d075e2d6a5c2948af99b8c71#B#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:12,960 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/9067bf598a3f49819b8d252b1a1d3323 is 50, key is test_row_0/B:col10/1732137252409/Put/seqid=0 2024-11-20T21:14:12,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741897_1073 (size=12983) 2024-11-20T21:14:12,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741898_1074 (size=12983) 2024-11-20T21:14:12,974 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/9727327d25f6455f94b2e3dfa0da9fdd as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/9727327d25f6455f94b2e3dfa0da9fdd 2024-11-20T21:14:12,982 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f3e2abb3d075e2d6a5c2948af99b8c71/A of f3e2abb3d075e2d6a5c2948af99b8c71 into 9727327d25f6455f94b2e3dfa0da9fdd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:12,983 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:12,983 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., storeName=f3e2abb3d075e2d6a5c2948af99b8c71/A, priority=12, startTime=1732137252920; duration=0sec 2024-11-20T21:14:12,983 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:12,983 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3e2abb3d075e2d6a5c2948af99b8c71:A 2024-11-20T21:14:12,983 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:14:12,985 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49600 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:14:12,985 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): f3e2abb3d075e2d6a5c2948af99b8c71/C is initiating minor compaction (all files) 2024-11-20T21:14:12,985 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3e2abb3d075e2d6a5c2948af99b8c71/C in TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:12,986 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/f48bb882d63d4494acfcb880e9d45ce9, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/e5ad822a561d4289a4d5c93468cccf58, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/54dbc05acd134e1b835a63fa2fc34b20, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/c624a12abf294e6f8fb00d950b9e802a] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp, totalSize=48.4 K 2024-11-20T21:14:12,986 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f48bb882d63d4494acfcb880e9d45ce9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732137249249 2024-11-20T21:14:12,987 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5ad822a561d4289a4d5c93468cccf58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732137251075 2024-11-20T21:14:12,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-20T21:14:12,987 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:14:12,987 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54dbc05acd134e1b835a63fa2fc34b20, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1732137251137 2024-11-20T21:14:12,988 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting c624a12abf294e6f8fb00d950b9e802a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1732137252409 2024-11-20T21:14:12,990 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:14:12,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-20T21:14:12,994 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:14:12,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T21:14:12,995 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:14:12,996 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:14:13,015 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3e2abb3d075e2d6a5c2948af99b8c71#C#compaction#62 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:13,015 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/3c929955d0ca49a6b1031772ecea6989 is 50, key is test_row_0/C:col10/1732137252409/Put/seqid=0 2024-11-20T21:14:13,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741899_1075 (size=12983) 2024-11-20T21:14:13,042 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/3c929955d0ca49a6b1031772ecea6989 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/3c929955d0ca49a6b1031772ecea6989 2024-11-20T21:14:13,055 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f3e2abb3d075e2d6a5c2948af99b8c71/C of f3e2abb3d075e2d6a5c2948af99b8c71 into 3c929955d0ca49a6b1031772ecea6989(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:13,055 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:13,055 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., storeName=f3e2abb3d075e2d6a5c2948af99b8c71/C, priority=12, startTime=1732137252920; duration=0sec 2024-11-20T21:14:13,056 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:13,057 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3e2abb3d075e2d6a5c2948af99b8c71:C 2024-11-20T21:14:13,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:13,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3e2abb3d075e2d6a5c2948af99b8c71 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-20T21:14:13,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=A 2024-11-20T21:14:13,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:13,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=B 2024-11-20T21:14:13,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:13,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=C 2024-11-20T21:14:13,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:13,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T21:14:13,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/f496ecc1f40a4ecc8deea43aa9663ac7 is 50, key is test_row_0/A:col10/1732137253088/Put/seqid=0 2024-11-20T21:14:13,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:13,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137313130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:13,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:13,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741900_1076 (size=14741) 2024-11-20T21:14:13,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:13,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:13,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137313134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:13,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137313131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:13,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:13,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:13,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:13,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137313134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:13,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:13,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137313134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:13,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,145 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:13,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:13,149 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/f496ecc1f40a4ecc8deea43aa9663ac7 2024-11-20T21:14:13,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-20T21:14:13,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:13,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:13,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:13,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:13,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:13,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:13,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/097c2f3277dc495e8befe412d9ac5f06 is 50, key is test_row_0/B:col10/1732137253088/Put/seqid=0 2024-11-20T21:14:13,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741901_1077 (size=12301) 2024-11-20T21:14:13,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/097c2f3277dc495e8befe412d9ac5f06 2024-11-20T21:14:13,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:13,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137313237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:13,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:13,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:13,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137313248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:13,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:13,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137313248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:13,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137313248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:13,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:13,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137313260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:13,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:13,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/245a520e0c0142209e5da2d926f44bd1 is 50, key is test_row_0/C:col10/1732137253088/Put/seqid=0 2024-11-20T21:14:13,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-20T21:14:13,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:13,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:13,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:13,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:13,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:13,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:13,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T21:14:13,308 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39687 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=0b16ab3f13b7,36987,1732137236536, table=TestAcidGuarantees, region=f3e2abb3d075e2d6a5c2948af99b8c71. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T21:14:13,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741902_1078 (size=12301) 2024-11-20T21:14:13,381 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/9067bf598a3f49819b8d252b1a1d3323 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/9067bf598a3f49819b8d252b1a1d3323 2024-11-20T21:14:13,391 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f3e2abb3d075e2d6a5c2948af99b8c71/B of f3e2abb3d075e2d6a5c2948af99b8c71 into 9067bf598a3f49819b8d252b1a1d3323(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:13,391 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:13,391 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., storeName=f3e2abb3d075e2d6a5c2948af99b8c71/B, priority=12, startTime=1732137252920; duration=0sec 2024-11-20T21:14:13,392 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:13,392 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3e2abb3d075e2d6a5c2948af99b8c71:B 2024-11-20T21:14:13,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:13,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137313447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:13,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,450 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:13,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-20T21:14:13,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:13,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:13,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:13,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:13,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:13,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:13,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:13,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137313469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:13,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:13,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137313469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:13,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:13,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137313469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:13,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:13,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,475 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:13,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:13,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137313477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:13,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:13,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-20T21:14:13,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T21:14:13,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:13,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:13,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:13,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:13,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:13,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:13,721 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/245a520e0c0142209e5da2d926f44bd1 2024-11-20T21:14:13,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/f496ecc1f40a4ecc8deea43aa9663ac7 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/f496ecc1f40a4ecc8deea43aa9663ac7 2024-11-20T21:14:13,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/f496ecc1f40a4ecc8deea43aa9663ac7, entries=200, sequenceid=416, filesize=14.4 K 2024-11-20T21:14:13,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/097c2f3277dc495e8befe412d9ac5f06 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/097c2f3277dc495e8befe412d9ac5f06 2024-11-20T21:14:13,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:13,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137313757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:13,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:13,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-20T21:14:13,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:13,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:13,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:13,779 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:13,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:13,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:13,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:13,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137313782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:13,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:13,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137313782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:13,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:13,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:13,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/097c2f3277dc495e8befe412d9ac5f06, entries=150, sequenceid=416, filesize=12.0 K 2024-11-20T21:14:13,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:13,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137313791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:13,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/245a520e0c0142209e5da2d926f44bd1 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/245a520e0c0142209e5da2d926f44bd1 2024-11-20T21:14:13,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:13,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:13,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137313793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:13,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/245a520e0c0142209e5da2d926f44bd1, entries=150, sequenceid=416, filesize=12.0 K 2024-11-20T21:14:13,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:13,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:13,806 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=114.05 KB/116790 for f3e2abb3d075e2d6a5c2948af99b8c71 in 715ms, sequenceid=416, compaction requested=false 2024-11-20T21:14:13,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:13,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-20T21:14:13,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:13,935 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing f3e2abb3d075e2d6a5c2948af99b8c71 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T21:14:13,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=A 2024-11-20T21:14:13,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:13,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=B 2024-11-20T21:14:13,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:13,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=C 2024-11-20T21:14:13,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:13,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/646a4ee96b7448dc8975bed4eb6ce686 is 50, key is test_row_0/A:col10/1732137253131/Put/seqid=0 2024-11-20T21:14:13,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741903_1079 (size=12301) 2024-11-20T21:14:13,983 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/646a4ee96b7448dc8975bed4eb6ce686 2024-11-20T21:14:13,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/ecd5bb6777484f548495b971c4d3fac2 is 50, key is test_row_0/B:col10/1732137253131/Put/seqid=0 2024-11-20T21:14:14,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741904_1080 (size=12301) 2024-11-20T21:14:14,022 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/ecd5bb6777484f548495b971c4d3fac2 2024-11-20T21:14:14,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/2ea17a718f1e4051a5f69e3b0b05237a is 50, key is test_row_0/C:col10/1732137253131/Put/seqid=0 2024-11-20T21:14:14,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741905_1081 (size=12301) 2024-11-20T21:14:14,070 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=437 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/2ea17a718f1e4051a5f69e3b0b05237a 2024-11-20T21:14:14,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/646a4ee96b7448dc8975bed4eb6ce686 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/646a4ee96b7448dc8975bed4eb6ce686 2024-11-20T21:14:14,099 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/646a4ee96b7448dc8975bed4eb6ce686, entries=150, sequenceid=437, filesize=12.0 K 2024-11-20T21:14:14,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/ecd5bb6777484f548495b971c4d3fac2 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/ecd5bb6777484f548495b971c4d3fac2 2024-11-20T21:14:14,120 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/ecd5bb6777484f548495b971c4d3fac2, entries=150, sequenceid=437, filesize=12.0 K 2024-11-20T21:14:14,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/2ea17a718f1e4051a5f69e3b0b05237a as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/2ea17a718f1e4051a5f69e3b0b05237a 2024-11-20T21:14:14,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T21:14:14,131 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/2ea17a718f1e4051a5f69e3b0b05237a, entries=150, sequenceid=437, filesize=12.0 K 2024-11-20T21:14:14,133 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for f3e2abb3d075e2d6a5c2948af99b8c71 in 198ms, sequenceid=437, compaction requested=true 2024-11-20T21:14:14,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:14,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:14,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-20T21:14:14,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-20T21:14:14,140 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-20T21:14:14,140 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1420 sec 2024-11-20T21:14:14,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.1510 sec 2024-11-20T21:14:14,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:14,309 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3e2abb3d075e2d6a5c2948af99b8c71 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T21:14:14,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=A 2024-11-20T21:14:14,309 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:14,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=B 2024-11-20T21:14:14,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:14,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=C 2024-11-20T21:14:14,310 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:14,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/221c45ab5ffa40e984f6d5d860b7cc32 is 50, key is test_row_0/A:col10/1732137254299/Put/seqid=0 2024-11-20T21:14:14,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741906_1082 (size=17181) 2024-11-20T21:14:14,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/221c45ab5ffa40e984f6d5d860b7cc32 2024-11-20T21:14:14,354 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=659, cellSize=30175, totalHeapSize=68776, min timestamp=1732137254318, max timestamp=1732137254351Num uniques 859; 2024-11-20T21:14:14,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:14,356 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=30175, suffix heap size=117064,new segment heap size=68776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=659 2024-11-20T21:14:14,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137314349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:14,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:14,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:14,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:14,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=659, cellSize=30175, totalHeapSize=68776, min timestamp=1732137254318, max timestamp=1732137254351Num uniques 859; 2024-11-20T21:14:14,359 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=30175, suffix heap size=117064,new segment heap size=68776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=659 2024-11-20T21:14:14,360 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=659, cellSize=30175, totalHeapSize=68776, min timestamp=1732137254318, max timestamp=1732137254351Num uniques 859; 2024-11-20T21:14:14,360 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=30175, suffix heap size=117064,new segment heap size=68776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=659 2024-11-20T21:14:14,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/702c556aa61f40d28e96bf272c11af2c is 50, key is test_row_0/B:col10/1732137254299/Put/seqid=0 2024-11-20T21:14:14,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137314368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:14,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:14,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:14,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:14,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:14,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137314370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:14,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:14,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:14,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:14,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:14,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137314375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:14,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:14,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:14,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137314375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:14,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:14,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:14,393 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:14,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741907_1083 (size=12301) 2024-11-20T21:14:14,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:14,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137314467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:14,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:14,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:14,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:14,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:14,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137314477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:14,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:14,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:14,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:14,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:14,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137314497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:14,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:14,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:14,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:14,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:14,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137314497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:14,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:14,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:14,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:14,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:14,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137314498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:14,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:14,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:14,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137314677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:14,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:14,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137314687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:14,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:14,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:14,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:14,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:14,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137314707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:14,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:14,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:14,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:14,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:14,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137314707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:14,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:14,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:14,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:14,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:14,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137314709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:14,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:14,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:14,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:14,829 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/702c556aa61f40d28e96bf272c11af2c 2024-11-20T21:14:14,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/9206ededa21845178b154bb2aa7c915e is 50, key is test_row_0/C:col10/1732137254299/Put/seqid=0 2024-11-20T21:14:14,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741908_1084 (size=12301) 2024-11-20T21:14:14,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/9206ededa21845178b154bb2aa7c915e 2024-11-20T21:14:14,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/221c45ab5ffa40e984f6d5d860b7cc32 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/221c45ab5ffa40e984f6d5d860b7cc32 2024-11-20T21:14:14,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/221c45ab5ffa40e984f6d5d860b7cc32, entries=250, sequenceid=448, filesize=16.8 K 2024-11-20T21:14:14,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/702c556aa61f40d28e96bf272c11af2c as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/702c556aa61f40d28e96bf272c11af2c 2024-11-20T21:14:14,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/702c556aa61f40d28e96bf272c11af2c, entries=150, sequenceid=448, filesize=12.0 K 2024-11-20T21:14:14,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/9206ededa21845178b154bb2aa7c915e as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/9206ededa21845178b154bb2aa7c915e 2024-11-20T21:14:14,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/9206ededa21845178b154bb2aa7c915e, entries=150, sequenceid=448, filesize=12.0 K 2024-11-20T21:14:14,992 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=140.89 KB/144270 for f3e2abb3d075e2d6a5c2948af99b8c71 in 683ms, sequenceid=448, compaction requested=true 2024-11-20T21:14:14,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:14,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3e2abb3d075e2d6a5c2948af99b8c71:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:14:14,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:14,992 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:14:14,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3e2abb3d075e2d6a5c2948af99b8c71:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:14:14,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:14,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3e2abb3d075e2d6a5c2948af99b8c71:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:14:14,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T21:14:14,993 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:14:14,995 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57206 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:14:14,995 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): f3e2abb3d075e2d6a5c2948af99b8c71/A is initiating minor compaction (all files) 2024-11-20T21:14:14,995 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3e2abb3d075e2d6a5c2948af99b8c71/A in TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:14,996 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/9727327d25f6455f94b2e3dfa0da9fdd, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/f496ecc1f40a4ecc8deea43aa9663ac7, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/646a4ee96b7448dc8975bed4eb6ce686, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/221c45ab5ffa40e984f6d5d860b7cc32] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp, totalSize=55.9 K 2024-11-20T21:14:14,997 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:14:14,997 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9727327d25f6455f94b2e3dfa0da9fdd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1732137252409 2024-11-20T21:14:14,997 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): f3e2abb3d075e2d6a5c2948af99b8c71/B is initiating minor compaction (all files) 2024-11-20T21:14:14,997 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3e2abb3d075e2d6a5c2948af99b8c71/B in TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:14,997 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/9067bf598a3f49819b8d252b1a1d3323, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/097c2f3277dc495e8befe412d9ac5f06, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/ecd5bb6777484f548495b971c4d3fac2, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/702c556aa61f40d28e96bf272c11af2c] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp, totalSize=48.7 K 2024-11-20T21:14:14,997 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f496ecc1f40a4ecc8deea43aa9663ac7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732137252753 2024-11-20T21:14:14,998 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 9067bf598a3f49819b8d252b1a1d3323, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1732137252409 2024-11-20T21:14:14,998 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 646a4ee96b7448dc8975bed4eb6ce686, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1732137253128 2024-11-20T21:14:14,998 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 097c2f3277dc495e8befe412d9ac5f06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732137252762 2024-11-20T21:14:14,998 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 221c45ab5ffa40e984f6d5d860b7cc32, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1732137254299 2024-11-20T21:14:14,999 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting ecd5bb6777484f548495b971c4d3fac2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1732137253128 2024-11-20T21:14:15,000 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 702c556aa61f40d28e96bf272c11af2c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1732137254299 2024-11-20T21:14:15,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3e2abb3d075e2d6a5c2948af99b8c71 3/3 column families, dataSize=147.60 KB heapSize=377.95 KB 2024-11-20T21:14:15,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=A 2024-11-20T21:14:15,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:14:15,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=B 2024-11-20T21:14:15,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:14:15,002 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=C 2024-11-20T21:14:15,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:14:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:15,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/624d75ecb13546cd921544b19c1eb687 is 50, key is test_row_0/A:col10/1732137254999/Put/seqid=0 2024-11-20T21:14:15,027 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3e2abb3d075e2d6a5c2948af99b8c71#A#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:15,028 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/46c71d3a8c084aafbe9864ee423c41e1 is 50, key is test_row_0/A:col10/1732137254299/Put/seqid=0 2024-11-20T21:14:15,041 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3e2abb3d075e2d6a5c2948af99b8c71#B#compaction#74 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:15,042 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/56d2ffcde1204b3b915c126ae8b684f5 is 50, key is test_row_0/B:col10/1732137254299/Put/seqid=0 2024-11-20T21:14:15,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:15,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137315030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:15,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:15,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:15,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137315037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:15,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137315037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:15,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:15,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:15,049 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137315038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:15,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:15,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:15,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137315044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:15,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:15,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741909_1085 (size=14741) 2024-11-20T21:14:15,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-20T21:14:15,137 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:14:15,138 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:14:15,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-20T21:14:15,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741910_1086 (size=13119) 2024-11-20T21:14:15,141 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:14:15,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T21:14:15,142 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:14:15,143 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:14:15,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741911_1087 (size=13119) 2024-11-20T21:14:15,153 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:15,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137315147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:15,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,154 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:15,158 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/56d2ffcde1204b3b915c126ae8b684f5 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/56d2ffcde1204b3b915c126ae8b684f5 2024-11-20T21:14:15,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:15,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137315157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:15,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:15,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:15,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137315157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:15,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:15,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:15,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137315157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:15,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:15,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,165 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:15,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137315158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:15,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,165 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:15,167 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f3e2abb3d075e2d6a5c2948af99b8c71/B of f3e2abb3d075e2d6a5c2948af99b8c71 into 56d2ffcde1204b3b915c126ae8b684f5(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:15,167 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:15,167 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., storeName=f3e2abb3d075e2d6a5c2948af99b8c71/B, priority=12, startTime=1732137254992; duration=0sec 2024-11-20T21:14:15,167 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:15,167 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3e2abb3d075e2d6a5c2948af99b8c71:B 2024-11-20T21:14:15,167 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:14:15,169 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:14:15,169 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): f3e2abb3d075e2d6a5c2948af99b8c71/C is initiating minor compaction (all files) 2024-11-20T21:14:15,169 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3e2abb3d075e2d6a5c2948af99b8c71/C in TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:15,169 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/3c929955d0ca49a6b1031772ecea6989, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/245a520e0c0142209e5da2d926f44bd1, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/2ea17a718f1e4051a5f69e3b0b05237a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/9206ededa21845178b154bb2aa7c915e] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp, totalSize=48.7 K 2024-11-20T21:14:15,169 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c929955d0ca49a6b1031772ecea6989, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1732137252409 2024-11-20T21:14:15,170 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 245a520e0c0142209e5da2d926f44bd1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1732137252762 2024-11-20T21:14:15,171 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ea17a718f1e4051a5f69e3b0b05237a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=437, earliestPutTs=1732137253128 2024-11-20T21:14:15,171 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 9206ededa21845178b154bb2aa7c915e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1732137254299 2024-11-20T21:14:15,194 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3e2abb3d075e2d6a5c2948af99b8c71#C#compaction#75 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:15,194 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/2c04abdb57084ecd825e33ad454fdca6 is 50, key is test_row_0/C:col10/1732137254299/Put/seqid=0 2024-11-20T21:14:15,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741912_1088 (size=13119) 2024-11-20T21:14:15,214 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/2c04abdb57084ecd825e33ad454fdca6 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/2c04abdb57084ecd825e33ad454fdca6 2024-11-20T21:14:15,223 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f3e2abb3d075e2d6a5c2948af99b8c71/C of f3e2abb3d075e2d6a5c2948af99b8c71 into 2c04abdb57084ecd825e33ad454fdca6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:15,223 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:15,223 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., storeName=f3e2abb3d075e2d6a5c2948af99b8c71/C, priority=12, startTime=1732137254993; duration=0sec 2024-11-20T21:14:15,223 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:15,224 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3e2abb3d075e2d6a5c2948af99b8c71:C 2024-11-20T21:14:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T21:14:15,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T21:14:15,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:15,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:15,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:15,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:15,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:15,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:15,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:15,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137315357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:15,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:15,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:15,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137315367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:15,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:15,372 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137315367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:15,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:15,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137315377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:15,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:15,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:15,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137315377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:15,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,380 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:15,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T21:14:15,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:15,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:15,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:15,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:15,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:15,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:15,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T21:14:15,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/624d75ecb13546cd921544b19c1eb687 2024-11-20T21:14:15,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/7d4aa33f8e534f289abc4e66128c54d6 is 50, key is test_row_0/B:col10/1732137254999/Put/seqid=0 2024-11-20T21:14:15,553 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/46c71d3a8c084aafbe9864ee423c41e1 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/46c71d3a8c084aafbe9864ee423c41e1 2024-11-20T21:14:15,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741913_1089 (size=12301) 2024-11-20T21:14:15,562 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f3e2abb3d075e2d6a5c2948af99b8c71/A of f3e2abb3d075e2d6a5c2948af99b8c71 into 46c71d3a8c084aafbe9864ee423c41e1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:15,562 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:15,562 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., storeName=f3e2abb3d075e2d6a5c2948af99b8c71/A, priority=12, startTime=1732137254992; duration=0sec 2024-11-20T21:14:15,562 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:15,562 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3e2abb3d075e2d6a5c2948af99b8c71:A 2024-11-20T21:14:15,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/7d4aa33f8e534f289abc4e66128c54d6 2024-11-20T21:14:15,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/9aaebe921716467ab17bb65aaaefcc3a is 50, key is test_row_0/C:col10/1732137254999/Put/seqid=0 2024-11-20T21:14:15,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T21:14:15,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741914_1090 (size=12301) 2024-11-20T21:14:15,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:15,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:15,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:15,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:15,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:15,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:15,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:15,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137315667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:15,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,670 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,671 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:15,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:15,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137315677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:15,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:15,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:15,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137315677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:15,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:15,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137315687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,689 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:15,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:15,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:15,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137315687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:15,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:15,691 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:15,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T21:14:15,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:15,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:15,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:15,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:15,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:15,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:15,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T21:14:15,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T21:14:15,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:15,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:15,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:15,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:15,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:15,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:16,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/9aaebe921716467ab17bb65aaaefcc3a 2024-11-20T21:14:16,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/624d75ecb13546cd921544b19c1eb687 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/624d75ecb13546cd921544b19c1eb687 2024-11-20T21:14:16,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/624d75ecb13546cd921544b19c1eb687, entries=200, sequenceid=477, filesize=14.4 K 2024-11-20T21:14:16,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/7d4aa33f8e534f289abc4e66128c54d6 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/7d4aa33f8e534f289abc4e66128c54d6 2024-11-20T21:14:16,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/7d4aa33f8e534f289abc4e66128c54d6, entries=150, sequenceid=477, filesize=12.0 K 2024-11-20T21:14:16,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/9aaebe921716467ab17bb65aaaefcc3a as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/9aaebe921716467ab17bb65aaaefcc3a 2024-11-20T21:14:16,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/9aaebe921716467ab17bb65aaaefcc3a, entries=150, sequenceid=477, filesize=12.0 K 2024-11-20T21:14:16,038 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~377.91 KB/386976, currentSize=46.96 KB/48090 for f3e2abb3d075e2d6a5c2948af99b8c71 in 1036ms, sequenceid=477, compaction requested=false 2024-11-20T21:14:16,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:16,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-20T21:14:16,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:16,071 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing f3e2abb3d075e2d6a5c2948af99b8c71 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T21:14:16,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=A 2024-11-20T21:14:16,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:16,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=B 2024-11-20T21:14:16,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:16,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=C 2024-11-20T21:14:16,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:16,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/6b2079a0f6334ee591a8e6aa07e4aec4 is 50, key is test_row_0/A:col10/1732137255036/Put/seqid=0 2024-11-20T21:14:16,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741915_1091 (size=12301) 2024-11-20T21:14:16,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:16,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:16,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732137256211, max timestamp=1732137256226Num uniques 859; 2024-11-20T21:14:16,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732137256211, max timestamp=1732137256226Num uniques 859; 2024-11-20T21:14:16,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T21:14:16,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T21:14:16,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732137256211, max timestamp=1732137256226Num uniques 859; 2024-11-20T21:14:16,230 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T21:14:16,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:16,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137316253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:16,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137316255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:16,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:16,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:16,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137316258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:16,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:16,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:16,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137316258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:16,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137316272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:16,273 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,274 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:16,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T21:14:16,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:16,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137316377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:16,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:16,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137316377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:16,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:16,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:16,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:16,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137316378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:16,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:16,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:16,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137316378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:16,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:16,384 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:16,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137316378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:16,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:16,512 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/6b2079a0f6334ee591a8e6aa07e4aec4 2024-11-20T21:14:16,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/310d199227cf4888bf4b421f9611c763 is 50, key is test_row_0/B:col10/1732137255036/Put/seqid=0 2024-11-20T21:14:16,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741916_1092 (size=12301) 2024-11-20T21:14:16,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:16,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137316586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:16,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:16,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137316586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:16,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:16,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:16,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137316587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:16,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:16,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137316587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:16,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:16,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:16,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137316587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:16,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:16,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:16,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:16,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:16,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137316897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:16,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:16,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:16,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137316897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:16,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:16,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:16,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137316897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:16,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:16,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:16,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137316898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:16,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:16,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:16,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137316898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:16,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:16,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:16,927 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/310d199227cf4888bf4b421f9611c763 2024-11-20T21:14:16,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/f045e25c3dcb419a8d4e7d3dab5ffdc3 is 50, key is test_row_0/C:col10/1732137255036/Put/seqid=0 2024-11-20T21:14:16,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741917_1093 (size=12301) 2024-11-20T21:14:16,944 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/f045e25c3dcb419a8d4e7d3dab5ffdc3 2024-11-20T21:14:16,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/6b2079a0f6334ee591a8e6aa07e4aec4 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/6b2079a0f6334ee591a8e6aa07e4aec4 2024-11-20T21:14:16,958 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/6b2079a0f6334ee591a8e6aa07e4aec4, entries=150, sequenceid=490, filesize=12.0 K 2024-11-20T21:14:16,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/310d199227cf4888bf4b421f9611c763 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/310d199227cf4888bf4b421f9611c763 2024-11-20T21:14:16,965 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/310d199227cf4888bf4b421f9611c763, entries=150, sequenceid=490, filesize=12.0 K 2024-11-20T21:14:16,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/f045e25c3dcb419a8d4e7d3dab5ffdc3 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/f045e25c3dcb419a8d4e7d3dab5ffdc3 2024-11-20T21:14:16,971 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/f045e25c3dcb419a8d4e7d3dab5ffdc3, entries=150, sequenceid=490, filesize=12.0 K 2024-11-20T21:14:16,972 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f3e2abb3d075e2d6a5c2948af99b8c71 in 902ms, sequenceid=490, compaction requested=true 2024-11-20T21:14:16,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:16,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:16,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-20T21:14:16,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-20T21:14:16,978 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-20T21:14:16,979 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8310 sec 2024-11-20T21:14:16,981 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.8410 sec 2024-11-20T21:14:17,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-20T21:14:17,297 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:14:17,298 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:14:17,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-20T21:14:17,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T21:14:17,300 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:14:17,301 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:14:17,301 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:14:17,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T21:14:17,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:17,412 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3e2abb3d075e2d6a5c2948af99b8c71 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-20T21:14:17,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=A 2024-11-20T21:14:17,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T21:14:17,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=B 2024-11-20T21:14:17,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T21:14:17,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=C 2024-11-20T21:14:17,413 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T21:14:17,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/0f0e8da225ae4835a78d4dccfd072f8e is 50, key is test_row_0/A:col10/1732137256252/Put/seqid=0 2024-11-20T21:14:17,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:17,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137317416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:17,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:17,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:17,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:17,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:17,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137317417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:17,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:17,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:17,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:17,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:17,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137317423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:17,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:17,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:17,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:17,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137317424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:17,431 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:17,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:17,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:17,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:17,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:17,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137317426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:17,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:17,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:17,432 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:17,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741918_1094 (size=17181) 2024-11-20T21:14:17,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T21:14:17,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:17,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:17,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:17,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:17,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:17,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:17,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:17,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137317537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:17,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:17,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:17,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:17,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:17,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137317537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:17,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:17,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:17,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:17,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:17,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137317537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:17,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:17,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:17,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137317537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:17,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:17,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:17,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:17,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:17,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:17,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T21:14:17,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:17,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:17,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:17,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:17,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:17,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:17,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T21:14:17,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:17,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137317747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:17,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:17,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:17,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:17,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137317747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:17,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:17,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:17,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:17,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:17,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:17,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137317747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:17,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:17,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:17,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137317748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:17,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:17,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:17,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:17,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:17,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:17,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T21:14:17,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:17,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:17,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:17,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:17,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:17,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:17,844 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/0f0e8da225ae4835a78d4dccfd072f8e 2024-11-20T21:14:17,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/7851c9ca1d9b4c14a976a057b0cf0e3d is 50, key is test_row_0/B:col10/1732137256252/Put/seqid=0 2024-11-20T21:14:17,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741919_1095 (size=12301) 2024-11-20T21:14:17,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/7851c9ca1d9b4c14a976a057b0cf0e3d 2024-11-20T21:14:17,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/18c9fdaf28b442e0a530b8d0798c1ad9 is 50, key is test_row_0/C:col10/1732137256252/Put/seqid=0 2024-11-20T21:14:17,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741920_1096 (size=12301) 2024-11-20T21:14:17,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T21:14:17,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:17,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:17,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:17,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:17,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:17,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:17,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T21:14:18,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:18,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137318057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:18,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:18,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:18,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:18,064 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:18,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137318058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:18,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:18,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:18,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:18,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:18,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137318058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:18,068 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:18,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137318060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:18,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T21:14:18,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:18,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:18,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:18,072 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:18,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:18,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:18,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:18,073 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:18,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:18,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T21:14:18,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:18,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:18,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:18,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:18,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:18,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:18,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=529 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/18c9fdaf28b442e0a530b8d0798c1ad9 2024-11-20T21:14:18,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/0f0e8da225ae4835a78d4dccfd072f8e as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/0f0e8da225ae4835a78d4dccfd072f8e 2024-11-20T21:14:18,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/0f0e8da225ae4835a78d4dccfd072f8e, entries=250, sequenceid=529, filesize=16.8 K 2024-11-20T21:14:18,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/7851c9ca1d9b4c14a976a057b0cf0e3d as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/7851c9ca1d9b4c14a976a057b0cf0e3d 2024-11-20T21:14:18,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/7851c9ca1d9b4c14a976a057b0cf0e3d, entries=150, sequenceid=529, filesize=12.0 K 2024-11-20T21:14:18,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/18c9fdaf28b442e0a530b8d0798c1ad9 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/18c9fdaf28b442e0a530b8d0798c1ad9 2024-11-20T21:14:18,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/18c9fdaf28b442e0a530b8d0798c1ad9, entries=150, sequenceid=529, filesize=12.0 K 2024-11-20T21:14:18,342 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~425.48 KB/435696, currentSize=33.54 KB/34350 for f3e2abb3d075e2d6a5c2948af99b8c71 in 930ms, sequenceid=529, compaction requested=true 2024-11-20T21:14:18,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:18,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3e2abb3d075e2d6a5c2948af99b8c71:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:14:18,343 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:14:18,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:18,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3e2abb3d075e2d6a5c2948af99b8c71:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:14:18,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:18,343 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:14:18,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3e2abb3d075e2d6a5c2948af99b8c71:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:14:18,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:18,344 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57342 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:14:18,345 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): f3e2abb3d075e2d6a5c2948af99b8c71/A is initiating minor compaction (all files) 2024-11-20T21:14:18,345 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3e2abb3d075e2d6a5c2948af99b8c71/A in TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:18,345 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/46c71d3a8c084aafbe9864ee423c41e1, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/624d75ecb13546cd921544b19c1eb687, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/6b2079a0f6334ee591a8e6aa07e4aec4, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/0f0e8da225ae4835a78d4dccfd072f8e] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp, totalSize=56.0 K 2024-11-20T21:14:18,345 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:14:18,346 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46c71d3a8c084aafbe9864ee423c41e1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1732137254299 2024-11-20T21:14:18,346 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): f3e2abb3d075e2d6a5c2948af99b8c71/B is initiating minor compaction (all files) 2024-11-20T21:14:18,346 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3e2abb3d075e2d6a5c2948af99b8c71/B in TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:18,346 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/56d2ffcde1204b3b915c126ae8b684f5, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/7d4aa33f8e534f289abc4e66128c54d6, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/310d199227cf4888bf4b421f9611c763, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/7851c9ca1d9b4c14a976a057b0cf0e3d] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp, totalSize=48.8 K 2024-11-20T21:14:18,346 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 56d2ffcde1204b3b915c126ae8b684f5, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1732137254299 2024-11-20T21:14:18,346 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 624d75ecb13546cd921544b19c1eb687, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1732137254366 2024-11-20T21:14:18,347 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d4aa33f8e534f289abc4e66128c54d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1732137254366 2024-11-20T21:14:18,347 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b2079a0f6334ee591a8e6aa07e4aec4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1732137255033 2024-11-20T21:14:18,348 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 310d199227cf4888bf4b421f9611c763, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1732137255033 2024-11-20T21:14:18,349 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0f0e8da225ae4835a78d4dccfd072f8e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=529, earliestPutTs=1732137256252 2024-11-20T21:14:18,349 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 7851c9ca1d9b4c14a976a057b0cf0e3d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=529, earliestPutTs=1732137256252 2024-11-20T21:14:18,368 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3e2abb3d075e2d6a5c2948af99b8c71#A#compaction#84 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:18,369 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3e2abb3d075e2d6a5c2948af99b8c71#B#compaction#85 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:18,369 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/1d1110a6aff840eaa58855bc4e543abe is 50, key is test_row_0/A:col10/1732137256252/Put/seqid=0 2024-11-20T21:14:18,370 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/66c0fd2fed0c429cb7473859d3b1d426 is 50, key is test_row_0/B:col10/1732137256252/Put/seqid=0 2024-11-20T21:14:18,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-20T21:14:18,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:18,390 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing f3e2abb3d075e2d6a5c2948af99b8c71 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-20T21:14:18,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=A 2024-11-20T21:14:18,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:18,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=B 2024-11-20T21:14:18,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:18,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=C 2024-11-20T21:14:18,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:18,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/0eeb9fd1253449249413bc97a42e78b1 is 50, key is test_row_1/A:col10/1732137257413/Put/seqid=0 2024-11-20T21:14:18,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741922_1098 (size=13255) 2024-11-20T21:14:18,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741921_1097 (size=13255) 2024-11-20T21:14:18,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T21:14:18,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:18,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:18,468 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/1d1110a6aff840eaa58855bc4e543abe as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/1d1110a6aff840eaa58855bc4e543abe 2024-11-20T21:14:18,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741923_1099 (size=9857) 2024-11-20T21:14:18,507 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f3e2abb3d075e2d6a5c2948af99b8c71/A of f3e2abb3d075e2d6a5c2948af99b8c71 into 1d1110a6aff840eaa58855bc4e543abe(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:18,508 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:18,508 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., storeName=f3e2abb3d075e2d6a5c2948af99b8c71/A, priority=12, startTime=1732137258342; duration=0sec 2024-11-20T21:14:18,508 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:18,508 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3e2abb3d075e2d6a5c2948af99b8c71:A 2024-11-20T21:14:18,508 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:14:18,510 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:14:18,510 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): f3e2abb3d075e2d6a5c2948af99b8c71/C is initiating minor compaction (all files) 2024-11-20T21:14:18,510 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3e2abb3d075e2d6a5c2948af99b8c71/C in TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:18,511 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/2c04abdb57084ecd825e33ad454fdca6, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/9aaebe921716467ab17bb65aaaefcc3a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/f045e25c3dcb419a8d4e7d3dab5ffdc3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/18c9fdaf28b442e0a530b8d0798c1ad9] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp, totalSize=48.8 K 2024-11-20T21:14:18,511 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c04abdb57084ecd825e33ad454fdca6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1732137254299 2024-11-20T21:14:18,511 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9aaebe921716467ab17bb65aaaefcc3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1732137254366 2024-11-20T21:14:18,512 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f045e25c3dcb419a8d4e7d3dab5ffdc3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1732137255033 2024-11-20T21:14:18,512 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18c9fdaf28b442e0a530b8d0798c1ad9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=529, earliestPutTs=1732137256252 2024-11-20T21:14:18,549 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3e2abb3d075e2d6a5c2948af99b8c71#C#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:18,550 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/6a07e27cadfe4a6daaf098566dcec149 is 50, key is test_row_0/C:col10/1732137256252/Put/seqid=0 2024-11-20T21:14:18,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741924_1100 (size=13255) 2024-11-20T21:14:18,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732137258568, max timestamp=1732137258585Num uniques 859; 2024-11-20T21:14:18,589 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732137258568, max timestamp=1732137258585Num uniques 859; 2024-11-20T21:14:18,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T21:14:18,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T21:14:18,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732137258568, max timestamp=1732137258585Num uniques 859; 2024-11-20T21:14:18,590 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T21:14:18,593 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/6a07e27cadfe4a6daaf098566dcec149 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/6a07e27cadfe4a6daaf098566dcec149 2024-11-20T21:14:18,602 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f3e2abb3d075e2d6a5c2948af99b8c71/C of f3e2abb3d075e2d6a5c2948af99b8c71 into 6a07e27cadfe4a6daaf098566dcec149(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:18,602 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:18,602 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., storeName=f3e2abb3d075e2d6a5c2948af99b8c71/C, priority=12, startTime=1732137258343; duration=0sec 2024-11-20T21:14:18,602 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:18,602 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3e2abb3d075e2d6a5c2948af99b8c71:C 2024-11-20T21:14:18,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9968, totalHeapSize=22912, min timestamp=1732137258612, max timestamp=1732137258632Num uniques 1018; 2024-11-20T21:14:18,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46608, new segment data size=9968, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-20T21:14:18,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9968, totalHeapSize=22912, min timestamp=1732137258612, max timestamp=1732137258632Num uniques 1018; 2024-11-20T21:14:18,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46608, new segment data size=9968, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-20T21:14:18,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9968, totalHeapSize=22912, min timestamp=1732137258612, max timestamp=1732137258632Num uniques 1018; 2024-11-20T21:14:18,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46608, new segment data size=9968, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-20T21:14:18,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732137258734, max timestamp=1732137258739Num uniques 1077; 2024-11-20T21:14:18,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-20T21:14:18,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732137258734, max timestamp=1732137258739Num uniques 1077; 2024-11-20T21:14:18,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732137258734, max timestamp=1732137258739Num uniques 1077; 2024-11-20T21:14:18,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-20T21:14:18,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-20T21:14:18,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:18,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137318781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:18,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732137258764, max timestamp=1732137258784Num uniques 1036; 2024-11-20T21:14:18,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-20T21:14:18,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:18,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:18,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732137258764, max timestamp=1732137258784Num uniques 1036; 2024-11-20T21:14:18,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:18,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-20T21:14:18,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732137258764, max timestamp=1732137258784Num uniques 1036; 2024-11-20T21:14:18,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=6870, suffix heap size=135712,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=150 2024-11-20T21:14:18,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18085, totalHeapSize=41320, min timestamp=1732137258809, max timestamp=1732137258830Num uniques 1095; 2024-11-20T21:14:18,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50145, new segment data size=18085, suffix heap size=141848,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=395 2024-11-20T21:14:18,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18090, totalHeapSize=41320, min timestamp=1732137258809, max timestamp=1732137258830Num uniques 1009; 2024-11-20T21:14:18,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18090, totalHeapSize=41320, min timestamp=1732137258809, max timestamp=1732137258830Num uniques 1095; 2024-11-20T21:14:18,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=18090, suffix heap size=132904,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=395 2024-11-20T21:14:18,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=18090, suffix heap size=141848,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=395 2024-11-20T21:14:18,846 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/66c0fd2fed0c429cb7473859d3b1d426 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/66c0fd2fed0c429cb7473859d3b1d426 2024-11-20T21:14:18,853 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f3e2abb3d075e2d6a5c2948af99b8c71/B of f3e2abb3d075e2d6a5c2948af99b8c71 into 66c0fd2fed0c429cb7473859d3b1d426(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:18,853 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:18,853 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., storeName=f3e2abb3d075e2d6a5c2948af99b8c71/B, priority=12, startTime=1732137258343; duration=0sec 2024-11-20T21:14:18,854 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:18,854 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3e2abb3d075e2d6a5c2948af99b8c71:B 2024-11-20T21:14:18,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:18,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137318859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:18,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:18,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:18,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:18,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137318866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:18,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:18,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137318867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:18,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:18,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:18,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137318870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:18,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:18,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:18,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:18,890 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/0eeb9fd1253449249413bc97a42e78b1 2024-11-20T21:14:18,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:18,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/47c431ee114547ff99c2eda29d874181 is 50, key is test_row_1/B:col10/1732137257413/Put/seqid=0 2024-11-20T21:14:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137318897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:18,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741925_1101 (size=9857) 2024-11-20T21:14:18,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:18,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137318978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:18,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:18,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:18,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:18,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137318978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:18,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:18,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:18,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:18,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:18,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137318979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:18,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137318980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:19,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137319107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:19,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:19,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:19,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137319187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:19,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:19,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137319187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:19,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:19,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137319187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:19,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:19,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:19,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:19,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137319187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:19,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:19,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:19,315 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/47c431ee114547ff99c2eda29d874181 2024-11-20T21:14:19,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/f03928eb5b9a4ac3a98efbc1c3fdeaa6 is 50, key is test_row_1/C:col10/1732137257413/Put/seqid=0 2024-11-20T21:14:19,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741926_1102 (size=9857) 2024-11-20T21:14:19,359 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/f03928eb5b9a4ac3a98efbc1c3fdeaa6 2024-11-20T21:14:19,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/0eeb9fd1253449249413bc97a42e78b1 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/0eeb9fd1253449249413bc97a42e78b1 2024-11-20T21:14:19,386 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/0eeb9fd1253449249413bc97a42e78b1, entries=100, sequenceid=537, filesize=9.6 K 2024-11-20T21:14:19,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/47c431ee114547ff99c2eda29d874181 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/47c431ee114547ff99c2eda29d874181 2024-11-20T21:14:19,412 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/47c431ee114547ff99c2eda29d874181, entries=100, sequenceid=537, filesize=9.6 K 2024-11-20T21:14:19,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:19,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137319421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:19,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/f03928eb5b9a4ac3a98efbc1c3fdeaa6 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/f03928eb5b9a4ac3a98efbc1c3fdeaa6 2024-11-20T21:14:19,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,425 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:19,445 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/f03928eb5b9a4ac3a98efbc1c3fdeaa6, entries=100, sequenceid=537, filesize=9.6 K 2024-11-20T21:14:19,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T21:14:19,449 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=154.31 KB/158010 for f3e2abb3d075e2d6a5c2948af99b8c71 in 1059ms, sequenceid=537, compaction requested=false 2024-11-20T21:14:19,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:19,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:19,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-20T21:14:19,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-20T21:14:19,455 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-20T21:14:19,455 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1520 sec 2024-11-20T21:14:19,459 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.1580 sec 2024-11-20T21:14:19,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:19,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3e2abb3d075e2d6a5c2948af99b8c71 3/3 column families, dataSize=167.72 KB heapSize=462.70 KB 2024-11-20T21:14:19,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1732137258857, max timestamp=1732137259509Num uniques 1254; 2024-11-20T21:14:19,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=57430, new segment data size=11630, suffix heap size=158384,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1254, new segment cells count=254 2024-11-20T21:14:19,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=A 2024-11-20T21:14:19,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:14:19,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=B 2024-11-20T21:14:19,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T21:14:19,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=C 2024-11-20T21:14:19,517 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T21:14:19,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/a5b0b29c5e1647b9a2c2084fa6038d9a is 50, key is test_row_0/A:col10/1732137259509/Put/seqid=0 2024-11-20T21:14:19,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:19,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137319536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:19,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:19,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137319537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:19,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:19,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:19,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:19,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137319537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:19,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:19,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137319537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:19,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:19,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:19,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741927_1103 (size=12301) 2024-11-20T21:14:19,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:19,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137319646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:19,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:19,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137319647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:19,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:19,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:19,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137319647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:19,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:19,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:19,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137319647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:19,657 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:19,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:19,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137319861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:19,866 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137319861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:19,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:19,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:19,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137319867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:19,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:19,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137319868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:19,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:19,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:19,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:19,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137319937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:19,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:19,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:19,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=647 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/a5b0b29c5e1647b9a2c2084fa6038d9a 2024-11-20T21:14:19,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/98a04fb4beb842e4baef7a159385f532 is 50, key is test_row_0/B:col10/1732137259509/Put/seqid=0 2024-11-20T21:14:20,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741928_1104 (size=12301) 2024-11-20T21:14:20,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:20,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137320179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:20,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:20,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137320179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:20,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:20,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137320179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:20,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:20,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:20,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:20,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:20,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137320181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:20,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:20,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:20,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:20,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:20,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:20,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:20,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:20,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:20,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:20,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=647 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/98a04fb4beb842e4baef7a159385f532 2024-11-20T21:14:20,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/75c829788db747c7a61716812bdf9c9a is 50, key is test_row_0/C:col10/1732137259509/Put/seqid=0 2024-11-20T21:14:20,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741929_1105 (size=12301) 2024-11-20T21:14:20,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=647 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/75c829788db747c7a61716812bdf9c9a 2024-11-20T21:14:20,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/a5b0b29c5e1647b9a2c2084fa6038d9a as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/a5b0b29c5e1647b9a2c2084fa6038d9a 2024-11-20T21:14:20,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/a5b0b29c5e1647b9a2c2084fa6038d9a, entries=150, sequenceid=647, filesize=12.0 K 2024-11-20T21:14:20,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/98a04fb4beb842e4baef7a159385f532 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/98a04fb4beb842e4baef7a159385f532 2024-11-20T21:14:20,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/98a04fb4beb842e4baef7a159385f532, entries=150, sequenceid=647, filesize=12.0 K 2024-11-20T21:14:20,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/75c829788db747c7a61716812bdf9c9a as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/75c829788db747c7a61716812bdf9c9a 2024-11-20T21:14:20,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/75c829788db747c7a61716812bdf9c9a, entries=150, sequenceid=647, filesize=12.0 K 2024-11-20T21:14:20,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~354.41 KB/362912, currentSize=53.67 KB/54960 for f3e2abb3d075e2d6a5c2948af99b8c71 in 1072ms, sequenceid=647, compaction requested=true 2024-11-20T21:14:20,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:20,584 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:20,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3e2abb3d075e2d6a5c2948af99b8c71:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:14:20,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:20,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3e2abb3d075e2d6a5c2948af99b8c71:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:14:20,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:20,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f3e2abb3d075e2d6a5c2948af99b8c71:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:14:20,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T21:14:20,586 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:20,586 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:20,586 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): f3e2abb3d075e2d6a5c2948af99b8c71/A is initiating minor compaction (all files) 2024-11-20T21:14:20,586 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3e2abb3d075e2d6a5c2948af99b8c71/A in TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:20,586 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/1d1110a6aff840eaa58855bc4e543abe, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/0eeb9fd1253449249413bc97a42e78b1, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/a5b0b29c5e1647b9a2c2084fa6038d9a] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp, totalSize=34.6 K 2024-11-20T21:14:20,587 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d1110a6aff840eaa58855bc4e543abe, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=529, earliestPutTs=1732137256252 2024-11-20T21:14:20,587 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 0eeb9fd1253449249413bc97a42e78b1, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1732137257413 2024-11-20T21:14:20,588 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting a5b0b29c5e1647b9a2c2084fa6038d9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=647, earliestPutTs=1732137259503 2024-11-20T21:14:20,588 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:20,588 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): f3e2abb3d075e2d6a5c2948af99b8c71/B is initiating minor compaction (all files) 2024-11-20T21:14:20,588 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3e2abb3d075e2d6a5c2948af99b8c71/B in TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:20,589 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/66c0fd2fed0c429cb7473859d3b1d426, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/47c431ee114547ff99c2eda29d874181, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/98a04fb4beb842e4baef7a159385f532] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp, totalSize=34.6 K 2024-11-20T21:14:20,590 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66c0fd2fed0c429cb7473859d3b1d426, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=529, earliestPutTs=1732137256252 2024-11-20T21:14:20,591 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47c431ee114547ff99c2eda29d874181, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1732137257413 2024-11-20T21:14:20,591 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98a04fb4beb842e4baef7a159385f532, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=647, earliestPutTs=1732137259503 2024-11-20T21:14:20,607 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3e2abb3d075e2d6a5c2948af99b8c71#A#compaction#93 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:20,608 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/ead28636377c4a9483edf97de5ce18fe is 50, key is test_row_0/A:col10/1732137259509/Put/seqid=0 2024-11-20T21:14:20,619 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3e2abb3d075e2d6a5c2948af99b8c71#B#compaction#94 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:20,620 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/193b0e69e7204b9c88a863202f56f2dd is 50, key is test_row_0/B:col10/1732137259509/Put/seqid=0 2024-11-20T21:14:20,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741930_1106 (size=13357) 2024-11-20T21:14:20,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741931_1107 (size=13357) 2024-11-20T21:14:20,643 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/193b0e69e7204b9c88a863202f56f2dd as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/193b0e69e7204b9c88a863202f56f2dd 2024-11-20T21:14:20,650 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3e2abb3d075e2d6a5c2948af99b8c71/B of f3e2abb3d075e2d6a5c2948af99b8c71 into 193b0e69e7204b9c88a863202f56f2dd(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:20,650 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:20,650 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., storeName=f3e2abb3d075e2d6a5c2948af99b8c71/B, priority=13, startTime=1732137260585; duration=0sec 2024-11-20T21:14:20,651 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:20,651 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3e2abb3d075e2d6a5c2948af99b8c71:B 2024-11-20T21:14:20,651 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:20,652 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:20,654 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): f3e2abb3d075e2d6a5c2948af99b8c71/C is initiating minor compaction (all files) 2024-11-20T21:14:20,654 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f3e2abb3d075e2d6a5c2948af99b8c71/C in TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:20,655 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/6a07e27cadfe4a6daaf098566dcec149, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/f03928eb5b9a4ac3a98efbc1c3fdeaa6, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/75c829788db747c7a61716812bdf9c9a] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp, totalSize=34.6 K 2024-11-20T21:14:20,655 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a07e27cadfe4a6daaf098566dcec149, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=529, earliestPutTs=1732137256252 2024-11-20T21:14:20,655 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f03928eb5b9a4ac3a98efbc1c3fdeaa6, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1732137257413 2024-11-20T21:14:20,656 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75c829788db747c7a61716812bdf9c9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=647, earliestPutTs=1732137259503 2024-11-20T21:14:20,669 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f3e2abb3d075e2d6a5c2948af99b8c71#C#compaction#95 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:20,670 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/6b33a08618534549aad707ea408e47f4 is 50, key is test_row_0/C:col10/1732137259509/Put/seqid=0 2024-11-20T21:14:20,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:20,689 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f3e2abb3d075e2d6a5c2948af99b8c71 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T21:14:20,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=A 2024-11-20T21:14:20,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:20,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=B 2024-11-20T21:14:20,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:20,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=C 2024-11-20T21:14:20,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:20,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741932_1108 (size=13357) 2024-11-20T21:14:20,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/bccf272c2ac84ea790f2af9b72dabad9 is 50, key is test_row_0/A:col10/1732137259535/Put/seqid=0 2024-11-20T21:14:20,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741933_1109 (size=17181) 2024-11-20T21:14:20,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:20,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137320721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:20,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:20,728 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:20,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137320723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:20,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:20,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:20,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:20,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:20,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:20,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:20,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137320724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:20,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:20,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137320725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:20,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:20,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:20,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:20,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:20,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:20,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:20,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:20,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137320837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:20,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:20,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:20,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:20,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:20,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137320837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:20,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:20,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:20,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:20,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:20,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137320837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:20,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:20,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:20,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:20,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137320837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:20,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:20,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:20,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:20,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:20,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:20,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41012 deadline: 1732137320946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:20,953 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 0b16ab3f13b7,36987,1732137236536 for 'test_row_0' in f3e2abb3d075e2d6a5c2948af99b8c71 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T21:14:20,956 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:20,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:20,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:21,042 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/ead28636377c4a9483edf97de5ce18fe as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/ead28636377c4a9483edf97de5ce18fe 2024-11-20T21:14:21,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:21,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137321046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:21,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:21,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:21,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:21,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:21,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137321047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:21,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:21,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:21,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:21,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:21,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137321047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:21,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:21,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137321047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:21,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:21,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:21,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:21,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:21,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:21,050 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3e2abb3d075e2d6a5c2948af99b8c71/A of f3e2abb3d075e2d6a5c2948af99b8c71 into ead28636377c4a9483edf97de5ce18fe(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:21,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:21,050 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:21,050 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., storeName=f3e2abb3d075e2d6a5c2948af99b8c71/A, priority=13, startTime=1732137260584; duration=0sec 2024-11-20T21:14:21,050 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:21,050 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3e2abb3d075e2d6a5c2948af99b8c71:A 2024-11-20T21:14:21,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=661 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/bccf272c2ac84ea790f2af9b72dabad9 2024-11-20T21:14:21,122 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/6b33a08618534549aad707ea408e47f4 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/6b33a08618534549aad707ea408e47f4 2024-11-20T21:14:21,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/0af203c9d16e41ae853a427d3b7774ad is 50, key is test_row_0/B:col10/1732137259535/Put/seqid=0 2024-11-20T21:14:21,143 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f3e2abb3d075e2d6a5c2948af99b8c71/C of f3e2abb3d075e2d6a5c2948af99b8c71 into 6b33a08618534549aad707ea408e47f4(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:21,143 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:21,143 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., storeName=f3e2abb3d075e2d6a5c2948af99b8c71/C, priority=13, startTime=1732137260585; duration=0sec 2024-11-20T21:14:21,143 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:21,143 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f3e2abb3d075e2d6a5c2948af99b8c71:C 2024-11-20T21:14:21,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741934_1110 (size=12301) 2024-11-20T21:14:21,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:21,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137321357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:21,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:21,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:21,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:21,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:21,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137321357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:21,359 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:21,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137321357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:21,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:21,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:21,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:21,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137321357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:21,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:21,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:21,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:21,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:21,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:21,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:21,360 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:21,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-20T21:14:21,473 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:14:21,473 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:14:21,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-20T21:14:21,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T21:14:21,476 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:14:21,477 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:14:21,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:14:21,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=661 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/0af203c9d16e41ae853a427d3b7774ad 2024-11-20T21:14:21,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/83a7703c3cbf4fdcbbf5abd181287427 is 50, key is test_row_0/C:col10/1732137259535/Put/seqid=0 2024-11-20T21:14:21,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T21:14:21,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741935_1111 (size=12301) 2024-11-20T21:14:21,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-20T21:14:21,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:21,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:21,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:21,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:21,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:21,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:21,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T21:14:21,806 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-20T21:14:21,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:21,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:21,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:21,806 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:21,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:21,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:21,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:21,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41018 deadline: 1732137321867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:21,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:21,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41022 deadline: 1732137321867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:21,869 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:21,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:21,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:21,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:21,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:21,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:21,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:40978 deadline: 1732137321868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:21,870 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:21,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:21,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:21,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:21,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:21,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41038 deadline: 1732137321869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:21,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:21,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:21,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:14:21,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-20T21:14:21,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:21,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:21,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:21,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:21,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:21,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:22,026 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=661 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/83a7703c3cbf4fdcbbf5abd181287427 2024-11-20T21:14:22,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/bccf272c2ac84ea790f2af9b72dabad9 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/bccf272c2ac84ea790f2af9b72dabad9 2024-11-20T21:14:22,063 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-20T21:14:22,064 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-20T21:14:22,064 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:14:22,064 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:14:22,066 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-20T21:14:22,066 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:14:22,069 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:22,069 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:22,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/bccf272c2ac84ea790f2af9b72dabad9, entries=250, sequenceid=661, filesize=16.8 K 2024-11-20T21:14:22,070 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:22,070 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:14:22,071 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:22,071 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:14:22,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/0af203c9d16e41ae853a427d3b7774ad as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/0af203c9d16e41ae853a427d3b7774ad 2024-11-20T21:14:22,071 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:22,071 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:22,072 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:14:22,072 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-20T21:14:22,072 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:14:22,072 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:22,072 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:22,072 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:14:22,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/0af203c9d16e41ae853a427d3b7774ad, entries=150, sequenceid=661, filesize=12.0 K 2024-11-20T21:14:22,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/83a7703c3cbf4fdcbbf5abd181287427 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/83a7703c3cbf4fdcbbf5abd181287427 2024-11-20T21:14:22,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/83a7703c3cbf4fdcbbf5abd181287427, entries=150, sequenceid=661, filesize=12.0 K 2024-11-20T21:14:22,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for f3e2abb3d075e2d6a5c2948af99b8c71 in 1416ms, sequenceid=661, compaction requested=false 2024-11-20T21:14:22,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:22,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T21:14:22,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-20T21:14:22,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:22,133 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing f3e2abb3d075e2d6a5c2948af99b8c71 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-20T21:14:22,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=A 2024-11-20T21:14:22,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:22,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=B 2024-11-20T21:14:22,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:22,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=C 2024-11-20T21:14:22,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:22,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/dc430b6c48d644cd96a21c48a6043f80 is 50, key is test_row_0/A:col10/1732137260723/Put/seqid=0 2024-11-20T21:14:22,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741936_1112 (size=12301) 2024-11-20T21:14:22,544 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=682 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/dc430b6c48d644cd96a21c48a6043f80 2024-11-20T21:14:22,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/3f90d07075604c869323d79b75a0e4f0 is 50, key is test_row_0/B:col10/1732137260723/Put/seqid=0 2024-11-20T21:14:22,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741937_1113 (size=12301) 2024-11-20T21:14:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T21:14:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:22,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. as already flushing 2024-11-20T21:14:22,878 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-20T21:14:22,878 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:14:22,878 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:22,878 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:22,878 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:14:22,889 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-20T21:14:22,889 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:14:22,889 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:22,889 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:22,889 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:14:22,889 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-20T21:14:22,889 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-20T21:14:22,889 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:14:22,889 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:14:22,889 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:22,889 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:22,889 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:22,889 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:22,889 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:14:22,889 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:14:22,958 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=682 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/3f90d07075604c869323d79b75a0e4f0 2024-11-20T21:14:22,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/ad9a6a9750d24d5aa51b37ca8bf9738d is 50, key is test_row_0/C:col10/1732137260723/Put/seqid=0 2024-11-20T21:14:22,969 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-20T21:14:22,969 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:14:22,969 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:22,969 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:22,969 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:14:22,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741938_1114 (size=12301) 2024-11-20T21:14:22,971 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=682 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/ad9a6a9750d24d5aa51b37ca8bf9738d 2024-11-20T21:14:22,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/dc430b6c48d644cd96a21c48a6043f80 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/dc430b6c48d644cd96a21c48a6043f80 2024-11-20T21:14:22,980 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/dc430b6c48d644cd96a21c48a6043f80, entries=150, sequenceid=682, filesize=12.0 K 2024-11-20T21:14:22,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/3f90d07075604c869323d79b75a0e4f0 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/3f90d07075604c869323d79b75a0e4f0 2024-11-20T21:14:22,984 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/3f90d07075604c869323d79b75a0e4f0, entries=150, sequenceid=682, filesize=12.0 K 2024-11-20T21:14:22,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/ad9a6a9750d24d5aa51b37ca8bf9738d as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/ad9a6a9750d24d5aa51b37ca8bf9738d 2024-11-20T21:14:22,989 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/ad9a6a9750d24d5aa51b37ca8bf9738d, entries=150, sequenceid=682, filesize=12.0 K 2024-11-20T21:14:22,990 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=33.54 KB/34350 for f3e2abb3d075e2d6a5c2948af99b8c71 in 857ms, sequenceid=682, compaction requested=true 2024-11-20T21:14:22,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for f3e2abb3d075e2d6a5c2948af99b8c71: 2024-11-20T21:14:22,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:22,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-20T21:14:22,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-20T21:14:22,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-20T21:14:22,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5150 sec 2024-11-20T21:14:22,996 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.5210 sec 2024-11-20T21:14:23,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-20T21:14:23,637 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:14:23,637 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T21:14:23,637 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 124 2024-11-20T21:14:23,637 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 116 2024-11-20T21:14:23,637 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 127 2024-11-20T21:14:23,637 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 112 2024-11-20T21:14:23,637 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 105 2024-11-20T21:14:23,637 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T21:14:23,637 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1940 2024-11-20T21:14:23,637 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1951 2024-11-20T21:14:23,637 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T21:14:23,637 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 922 2024-11-20T21:14:23,638 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2752 rows 2024-11-20T21:14:23,638 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 920 2024-11-20T21:14:23,638 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2749 rows 2024-11-20T21:14:23,638 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T21:14:23,638 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T21:14:23,638 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:23,638 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:23,638 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:14:23,642 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-20T21:14:23,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T21:14:23,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-20T21:14:23,650 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732137263650"}]},"ts":"1732137263650"} 2024-11-20T21:14:23,652 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T21:14:23,652 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T21:14:23,654 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T21:14:23,658 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3e2abb3d075e2d6a5c2948af99b8c71, UNASSIGN}] 2024-11-20T21:14:23,658 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3e2abb3d075e2d6a5c2948af99b8c71, UNASSIGN 2024-11-20T21:14:23,660 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=f3e2abb3d075e2d6a5c2948af99b8c71, regionState=CLOSING, regionLocation=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:23,660 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39687 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=CLOSING, location=0b16ab3f13b7,36987,1732137236536, table=TestAcidGuarantees, region=f3e2abb3d075e2d6a5c2948af99b8c71. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T21:14:23,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3e2abb3d075e2d6a5c2948af99b8c71, UNASSIGN because future has completed 2024-11-20T21:14:23,663 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T21:14:23,663 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536}] 2024-11-20T21:14:23,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-20T21:14:23,819 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:23,819 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T21:14:23,820 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing f3e2abb3d075e2d6a5c2948af99b8c71, disabling compactions & flushes 2024-11-20T21:14:23,820 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:23,820 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:23,820 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. after waiting 0 ms 2024-11-20T21:14:23,820 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:23,820 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing f3e2abb3d075e2d6a5c2948af99b8c71 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-20T21:14:23,821 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=A 2024-11-20T21:14:23,821 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:23,821 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=B 2024-11-20T21:14:23,821 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:23,821 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f3e2abb3d075e2d6a5c2948af99b8c71, store=C 2024-11-20T21:14:23,821 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:23,826 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/075a874c37174bf4b2d6f7223c27a5ac is 50, key is test_row_0/A:col10/1732137262967/Put/seqid=0 2024-11-20T21:14:23,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741939_1115 (size=12301) 2024-11-20T21:14:23,830 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=690 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/075a874c37174bf4b2d6f7223c27a5ac 2024-11-20T21:14:23,838 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/d23be0ddcfe74852855ac8f188268973 is 50, key is test_row_0/B:col10/1732137262967/Put/seqid=0 2024-11-20T21:14:23,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741940_1116 (size=12301) 2024-11-20T21:14:23,847 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=690 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/d23be0ddcfe74852855ac8f188268973 2024-11-20T21:14:23,854 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/168c757e3e8d44a0a0d966b683e1a51f is 50, key is test_row_0/C:col10/1732137262967/Put/seqid=0 2024-11-20T21:14:23,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741941_1117 (size=12301) 2024-11-20T21:14:23,860 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=690 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/168c757e3e8d44a0a0d966b683e1a51f 2024-11-20T21:14:23,864 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/A/075a874c37174bf4b2d6f7223c27a5ac as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/075a874c37174bf4b2d6f7223c27a5ac 2024-11-20T21:14:23,869 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/075a874c37174bf4b2d6f7223c27a5ac, entries=150, sequenceid=690, filesize=12.0 K 2024-11-20T21:14:23,870 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/B/d23be0ddcfe74852855ac8f188268973 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/d23be0ddcfe74852855ac8f188268973 2024-11-20T21:14:23,875 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/d23be0ddcfe74852855ac8f188268973, entries=150, sequenceid=690, filesize=12.0 K 2024-11-20T21:14:23,876 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/.tmp/C/168c757e3e8d44a0a0d966b683e1a51f as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/168c757e3e8d44a0a0d966b683e1a51f 2024-11-20T21:14:23,881 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/168c757e3e8d44a0a0d966b683e1a51f, entries=150, sequenceid=690, filesize=12.0 K 2024-11-20T21:14:23,882 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f3e2abb3d075e2d6a5c2948af99b8c71 in 62ms, sequenceid=690, compaction requested=true 2024-11-20T21:14:23,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/51d5a0c139024693af6992fbbc420165, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/483e0ac434054a63b4ee1a1cc9527295, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/ce8776eae5cc4c16a94ff44e02ebea2b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/b5e5077ca22a4974b765f82fcb07c3e4, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/b890b621823642d2bf0a83a6cc8ffd18, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/9721e867538f42588695109ee4c91a56, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/bfa49b73e1874c3a8ea30cfd2050bd75, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/b08c4f2dc6b54db58410ff48604a6054, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/faba36ff60c54630bd5e2e9deb4323f0, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/ed64ac5ca4714faa9d7289544b413a44, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/126c46818dc7429fbba3563004f312c7, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/5c1561a7eadc4df0a07fb29e1c2e0b01, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/971af64ed749470b90164f46ce7b1bc1, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/e292dbdc37614c298748c1c41f55b9cf, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/68ccace2bd3949208c2a4717809293ff, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/4320625392424c16b1d4e5ddb22b6d15, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/5e747b9214544baf8bf29445ae449995, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/ebd858b725474d16b232ace1e4816e44, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/7e910931a5b94696bfe300ed036ca4e6, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/9727327d25f6455f94b2e3dfa0da9fdd, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/754ee91f2d7140158d7d976e750cd74f, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/f496ecc1f40a4ecc8deea43aa9663ac7, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/646a4ee96b7448dc8975bed4eb6ce686, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/221c45ab5ffa40e984f6d5d860b7cc32, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/46c71d3a8c084aafbe9864ee423c41e1, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/624d75ecb13546cd921544b19c1eb687, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/6b2079a0f6334ee591a8e6aa07e4aec4, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/0f0e8da225ae4835a78d4dccfd072f8e, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/1d1110a6aff840eaa58855bc4e543abe, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/0eeb9fd1253449249413bc97a42e78b1, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/a5b0b29c5e1647b9a2c2084fa6038d9a] to archive 2024-11-20T21:14:23,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T21:14:23,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/51d5a0c139024693af6992fbbc420165 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/51d5a0c139024693af6992fbbc420165 2024-11-20T21:14:23,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/483e0ac434054a63b4ee1a1cc9527295 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/483e0ac434054a63b4ee1a1cc9527295 2024-11-20T21:14:23,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/ce8776eae5cc4c16a94ff44e02ebea2b to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/ce8776eae5cc4c16a94ff44e02ebea2b 2024-11-20T21:14:23,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/b5e5077ca22a4974b765f82fcb07c3e4 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/b5e5077ca22a4974b765f82fcb07c3e4 2024-11-20T21:14:23,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/b890b621823642d2bf0a83a6cc8ffd18 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/b890b621823642d2bf0a83a6cc8ffd18 2024-11-20T21:14:23,898 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/9721e867538f42588695109ee4c91a56 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/9721e867538f42588695109ee4c91a56 2024-11-20T21:14:23,899 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/bfa49b73e1874c3a8ea30cfd2050bd75 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/bfa49b73e1874c3a8ea30cfd2050bd75 2024-11-20T21:14:23,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/b08c4f2dc6b54db58410ff48604a6054 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/b08c4f2dc6b54db58410ff48604a6054 2024-11-20T21:14:23,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/faba36ff60c54630bd5e2e9deb4323f0 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/faba36ff60c54630bd5e2e9deb4323f0 2024-11-20T21:14:23,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/ed64ac5ca4714faa9d7289544b413a44 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/ed64ac5ca4714faa9d7289544b413a44 2024-11-20T21:14:23,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/126c46818dc7429fbba3563004f312c7 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/126c46818dc7429fbba3563004f312c7 2024-11-20T21:14:23,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/5c1561a7eadc4df0a07fb29e1c2e0b01 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/5c1561a7eadc4df0a07fb29e1c2e0b01 2024-11-20T21:14:23,910 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/971af64ed749470b90164f46ce7b1bc1 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/971af64ed749470b90164f46ce7b1bc1 2024-11-20T21:14:23,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/e292dbdc37614c298748c1c41f55b9cf to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/e292dbdc37614c298748c1c41f55b9cf 2024-11-20T21:14:23,913 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/68ccace2bd3949208c2a4717809293ff to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/68ccace2bd3949208c2a4717809293ff 2024-11-20T21:14:23,915 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/4320625392424c16b1d4e5ddb22b6d15 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/4320625392424c16b1d4e5ddb22b6d15 2024-11-20T21:14:23,916 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/5e747b9214544baf8bf29445ae449995 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/5e747b9214544baf8bf29445ae449995 2024-11-20T21:14:23,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/ebd858b725474d16b232ace1e4816e44 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/ebd858b725474d16b232ace1e4816e44 2024-11-20T21:14:23,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/7e910931a5b94696bfe300ed036ca4e6 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/7e910931a5b94696bfe300ed036ca4e6 2024-11-20T21:14:23,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/9727327d25f6455f94b2e3dfa0da9fdd to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/9727327d25f6455f94b2e3dfa0da9fdd 2024-11-20T21:14:23,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/754ee91f2d7140158d7d976e750cd74f to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/754ee91f2d7140158d7d976e750cd74f 2024-11-20T21:14:23,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/f496ecc1f40a4ecc8deea43aa9663ac7 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/f496ecc1f40a4ecc8deea43aa9663ac7 2024-11-20T21:14:23,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/646a4ee96b7448dc8975bed4eb6ce686 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/646a4ee96b7448dc8975bed4eb6ce686 2024-11-20T21:14:23,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/221c45ab5ffa40e984f6d5d860b7cc32 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/221c45ab5ffa40e984f6d5d860b7cc32 2024-11-20T21:14:23,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/46c71d3a8c084aafbe9864ee423c41e1 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/46c71d3a8c084aafbe9864ee423c41e1 2024-11-20T21:14:23,928 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/624d75ecb13546cd921544b19c1eb687 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/624d75ecb13546cd921544b19c1eb687 2024-11-20T21:14:23,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/6b2079a0f6334ee591a8e6aa07e4aec4 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/6b2079a0f6334ee591a8e6aa07e4aec4 2024-11-20T21:14:23,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/0f0e8da225ae4835a78d4dccfd072f8e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/0f0e8da225ae4835a78d4dccfd072f8e 2024-11-20T21:14:23,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/1d1110a6aff840eaa58855bc4e543abe to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/1d1110a6aff840eaa58855bc4e543abe 2024-11-20T21:14:23,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/0eeb9fd1253449249413bc97a42e78b1 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/0eeb9fd1253449249413bc97a42e78b1 2024-11-20T21:14:23,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/a5b0b29c5e1647b9a2c2084fa6038d9a to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/a5b0b29c5e1647b9a2c2084fa6038d9a 2024-11-20T21:14:23,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/ae2f1f399b174c5aac8432b0d0e887bb, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/dc26e315658649be831aefcf744b8066, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/a2e6ac5c2ca74354bf47eb626718245d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/daac3169ebcf42a5b8224b0d3972dcf8, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/90d105a5f2d043a1b9646ed5c5c6e27d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/5e328f08584b4d2ca5a09b7297032fcc, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/6c4ffe3b88624b4aba5c2592aa696245, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/128b16cc6a7b443089162937fc0acfee, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/1048ff9dcb2b4bbaafd078435c3b6c59, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/0d533027012e4fad813b27442832038b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/e4accd85f5714eb587ff612fb415df1c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/d9c2b1b13f50484695194beb69f9b69c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/c363e517351f4218a59ada1fa15bd104, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/a3e50a943b1a4cdf8cf5976f9f89018d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/0803397ccc4f43ccb64789985598e618, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/a15e758efa194486807468c0f1d296eb, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/e43b83647ea3436c99d884b2ebdb06b4, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/86d5e90fc0a7447f8ea3c95cdc4399fd, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/f0e0cd6192db477f95fffcc87f23f067, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/9067bf598a3f49819b8d252b1a1d3323, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/c5bdb08e67794f1e9642f35098bfe58b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/097c2f3277dc495e8befe412d9ac5f06, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/ecd5bb6777484f548495b971c4d3fac2, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/56d2ffcde1204b3b915c126ae8b684f5, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/702c556aa61f40d28e96bf272c11af2c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/7d4aa33f8e534f289abc4e66128c54d6, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/310d199227cf4888bf4b421f9611c763, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/66c0fd2fed0c429cb7473859d3b1d426, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/7851c9ca1d9b4c14a976a057b0cf0e3d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/47c431ee114547ff99c2eda29d874181, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/98a04fb4beb842e4baef7a159385f532] to archive 2024-11-20T21:14:23,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T21:14:23,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/ae2f1f399b174c5aac8432b0d0e887bb to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/ae2f1f399b174c5aac8432b0d0e887bb 2024-11-20T21:14:23,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/dc26e315658649be831aefcf744b8066 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/dc26e315658649be831aefcf744b8066 2024-11-20T21:14:23,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/a2e6ac5c2ca74354bf47eb626718245d to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/a2e6ac5c2ca74354bf47eb626718245d 2024-11-20T21:14:23,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/daac3169ebcf42a5b8224b0d3972dcf8 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/daac3169ebcf42a5b8224b0d3972dcf8 2024-11-20T21:14:23,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/90d105a5f2d043a1b9646ed5c5c6e27d to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/90d105a5f2d043a1b9646ed5c5c6e27d 2024-11-20T21:14:23,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/5e328f08584b4d2ca5a09b7297032fcc to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/5e328f08584b4d2ca5a09b7297032fcc 2024-11-20T21:14:23,961 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/6c4ffe3b88624b4aba5c2592aa696245 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/6c4ffe3b88624b4aba5c2592aa696245 2024-11-20T21:14:23,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/128b16cc6a7b443089162937fc0acfee to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/128b16cc6a7b443089162937fc0acfee 2024-11-20T21:14:23,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/1048ff9dcb2b4bbaafd078435c3b6c59 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/1048ff9dcb2b4bbaafd078435c3b6c59 2024-11-20T21:14:23,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/0d533027012e4fad813b27442832038b to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/0d533027012e4fad813b27442832038b 2024-11-20T21:14:23,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-20T21:14:23,968 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/e4accd85f5714eb587ff612fb415df1c to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/e4accd85f5714eb587ff612fb415df1c 2024-11-20T21:14:23,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/d9c2b1b13f50484695194beb69f9b69c to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/d9c2b1b13f50484695194beb69f9b69c 2024-11-20T21:14:23,971 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/c363e517351f4218a59ada1fa15bd104 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/c363e517351f4218a59ada1fa15bd104 2024-11-20T21:14:23,972 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/a3e50a943b1a4cdf8cf5976f9f89018d to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/a3e50a943b1a4cdf8cf5976f9f89018d 2024-11-20T21:14:23,974 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/0803397ccc4f43ccb64789985598e618 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/0803397ccc4f43ccb64789985598e618 2024-11-20T21:14:23,975 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/a15e758efa194486807468c0f1d296eb to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/a15e758efa194486807468c0f1d296eb 2024-11-20T21:14:23,976 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/e43b83647ea3436c99d884b2ebdb06b4 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/e43b83647ea3436c99d884b2ebdb06b4 2024-11-20T21:14:23,978 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/86d5e90fc0a7447f8ea3c95cdc4399fd to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/86d5e90fc0a7447f8ea3c95cdc4399fd 2024-11-20T21:14:23,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/f0e0cd6192db477f95fffcc87f23f067 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/f0e0cd6192db477f95fffcc87f23f067 2024-11-20T21:14:23,986 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/9067bf598a3f49819b8d252b1a1d3323 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/9067bf598a3f49819b8d252b1a1d3323 2024-11-20T21:14:23,988 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/c5bdb08e67794f1e9642f35098bfe58b to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/c5bdb08e67794f1e9642f35098bfe58b 2024-11-20T21:14:23,989 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/097c2f3277dc495e8befe412d9ac5f06 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/097c2f3277dc495e8befe412d9ac5f06 2024-11-20T21:14:23,991 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/ecd5bb6777484f548495b971c4d3fac2 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/ecd5bb6777484f548495b971c4d3fac2 2024-11-20T21:14:23,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/56d2ffcde1204b3b915c126ae8b684f5 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/56d2ffcde1204b3b915c126ae8b684f5 2024-11-20T21:14:23,994 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/702c556aa61f40d28e96bf272c11af2c to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/702c556aa61f40d28e96bf272c11af2c 2024-11-20T21:14:23,995 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/7d4aa33f8e534f289abc4e66128c54d6 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/7d4aa33f8e534f289abc4e66128c54d6 2024-11-20T21:14:23,996 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/310d199227cf4888bf4b421f9611c763 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/310d199227cf4888bf4b421f9611c763 2024-11-20T21:14:23,997 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/66c0fd2fed0c429cb7473859d3b1d426 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/66c0fd2fed0c429cb7473859d3b1d426 2024-11-20T21:14:23,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/7851c9ca1d9b4c14a976a057b0cf0e3d to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/7851c9ca1d9b4c14a976a057b0cf0e3d 2024-11-20T21:14:23,999 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/47c431ee114547ff99c2eda29d874181 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/47c431ee114547ff99c2eda29d874181 2024-11-20T21:14:24,001 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/98a04fb4beb842e4baef7a159385f532 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/98a04fb4beb842e4baef7a159385f532 2024-11-20T21:14:24,002 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/9b35e56cec954e0ebf21d63658b0d5c9, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/10d1eec311054a6b858b56c7fc0c9c0b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/0320de669d494778b6333c6e5e95bb7d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/fcd83eca2d774303a260154cae1c5b26, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/ffd2b6e61ce043f08bc4f1ce13685c6a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/1179a7b2c21c49f6a64e109149d25990, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/49f1588be7574800b59ffd075b9824fa, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/062acbfbec1348b7a9670ae189d97a6b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/87e6fe77c77b4affbcd3bd0d03110042, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/17ef6a74624746678088a1ce033b0352, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/2999dfbb933443bebb1121dabbe9b755, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/aefa9772995248aca4ebb688cfb04002, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/df92c60a5ec94b29b886d7a7415cff4a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/920ab2711bda41eabfa693d8171e5e11, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/1e4b397403084f5e87630f51a06a1beb, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/f48bb882d63d4494acfcb880e9d45ce9, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/45597aca93e94ff28dc49f37cfa89a7e, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/e5ad822a561d4289a4d5c93468cccf58, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/54dbc05acd134e1b835a63fa2fc34b20, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/3c929955d0ca49a6b1031772ecea6989, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/c624a12abf294e6f8fb00d950b9e802a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/245a520e0c0142209e5da2d926f44bd1, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/2ea17a718f1e4051a5f69e3b0b05237a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/2c04abdb57084ecd825e33ad454fdca6, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/9206ededa21845178b154bb2aa7c915e, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/9aaebe921716467ab17bb65aaaefcc3a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/f045e25c3dcb419a8d4e7d3dab5ffdc3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/6a07e27cadfe4a6daaf098566dcec149, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/18c9fdaf28b442e0a530b8d0798c1ad9, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/f03928eb5b9a4ac3a98efbc1c3fdeaa6, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/75c829788db747c7a61716812bdf9c9a] to archive 2024-11-20T21:14:24,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T21:14:24,005 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/9b35e56cec954e0ebf21d63658b0d5c9 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/9b35e56cec954e0ebf21d63658b0d5c9 2024-11-20T21:14:24,006 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/10d1eec311054a6b858b56c7fc0c9c0b to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/10d1eec311054a6b858b56c7fc0c9c0b 2024-11-20T21:14:24,007 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/0320de669d494778b6333c6e5e95bb7d to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/0320de669d494778b6333c6e5e95bb7d 2024-11-20T21:14:24,008 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/fcd83eca2d774303a260154cae1c5b26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/fcd83eca2d774303a260154cae1c5b26 2024-11-20T21:14:24,009 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/ffd2b6e61ce043f08bc4f1ce13685c6a to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/ffd2b6e61ce043f08bc4f1ce13685c6a 2024-11-20T21:14:24,010 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/1179a7b2c21c49f6a64e109149d25990 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/1179a7b2c21c49f6a64e109149d25990 2024-11-20T21:14:24,011 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/49f1588be7574800b59ffd075b9824fa to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/49f1588be7574800b59ffd075b9824fa 2024-11-20T21:14:24,012 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/062acbfbec1348b7a9670ae189d97a6b to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/062acbfbec1348b7a9670ae189d97a6b 2024-11-20T21:14:24,013 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/87e6fe77c77b4affbcd3bd0d03110042 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/87e6fe77c77b4affbcd3bd0d03110042 2024-11-20T21:14:24,014 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/17ef6a74624746678088a1ce033b0352 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/17ef6a74624746678088a1ce033b0352 2024-11-20T21:14:24,015 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/2999dfbb933443bebb1121dabbe9b755 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/2999dfbb933443bebb1121dabbe9b755 2024-11-20T21:14:24,016 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/aefa9772995248aca4ebb688cfb04002 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/aefa9772995248aca4ebb688cfb04002 2024-11-20T21:14:24,018 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/df92c60a5ec94b29b886d7a7415cff4a to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/df92c60a5ec94b29b886d7a7415cff4a 2024-11-20T21:14:24,019 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/920ab2711bda41eabfa693d8171e5e11 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/920ab2711bda41eabfa693d8171e5e11 2024-11-20T21:14:24,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/1e4b397403084f5e87630f51a06a1beb to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/1e4b397403084f5e87630f51a06a1beb 2024-11-20T21:14:24,021 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/f48bb882d63d4494acfcb880e9d45ce9 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/f48bb882d63d4494acfcb880e9d45ce9 2024-11-20T21:14:24,022 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/45597aca93e94ff28dc49f37cfa89a7e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/45597aca93e94ff28dc49f37cfa89a7e 2024-11-20T21:14:24,023 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/e5ad822a561d4289a4d5c93468cccf58 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/e5ad822a561d4289a4d5c93468cccf58 2024-11-20T21:14:24,024 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/54dbc05acd134e1b835a63fa2fc34b20 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/54dbc05acd134e1b835a63fa2fc34b20 2024-11-20T21:14:24,025 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/3c929955d0ca49a6b1031772ecea6989 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/3c929955d0ca49a6b1031772ecea6989 2024-11-20T21:14:24,026 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/c624a12abf294e6f8fb00d950b9e802a to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/c624a12abf294e6f8fb00d950b9e802a 2024-11-20T21:14:24,027 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/245a520e0c0142209e5da2d926f44bd1 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/245a520e0c0142209e5da2d926f44bd1 2024-11-20T21:14:24,029 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/2ea17a718f1e4051a5f69e3b0b05237a to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/2ea17a718f1e4051a5f69e3b0b05237a 2024-11-20T21:14:24,030 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/2c04abdb57084ecd825e33ad454fdca6 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/2c04abdb57084ecd825e33ad454fdca6 2024-11-20T21:14:24,031 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/9206ededa21845178b154bb2aa7c915e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/9206ededa21845178b154bb2aa7c915e 2024-11-20T21:14:24,033 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/9aaebe921716467ab17bb65aaaefcc3a to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/9aaebe921716467ab17bb65aaaefcc3a 2024-11-20T21:14:24,034 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/f045e25c3dcb419a8d4e7d3dab5ffdc3 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/f045e25c3dcb419a8d4e7d3dab5ffdc3 2024-11-20T21:14:24,035 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/6a07e27cadfe4a6daaf098566dcec149 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/6a07e27cadfe4a6daaf098566dcec149 2024-11-20T21:14:24,039 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/18c9fdaf28b442e0a530b8d0798c1ad9 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/18c9fdaf28b442e0a530b8d0798c1ad9 2024-11-20T21:14:24,040 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/f03928eb5b9a4ac3a98efbc1c3fdeaa6 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/f03928eb5b9a4ac3a98efbc1c3fdeaa6 2024-11-20T21:14:24,042 DEBUG [StoreCloser-TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/75c829788db747c7a61716812bdf9c9a to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/75c829788db747c7a61716812bdf9c9a 2024-11-20T21:14:24,048 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/recovered.edits/693.seqid, newMaxSeqId=693, maxSeqId=1 2024-11-20T21:14:24,048 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. 2024-11-20T21:14:24,049 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for f3e2abb3d075e2d6a5c2948af99b8c71: Waiting for close lock at 1732137263820Running coprocessor pre-close hooks at 1732137263820Disabling compacts and flushes for region at 1732137263820Disabling writes for close at 1732137263820Obtaining lock to block concurrent updates at 1732137263820Preparing flush snapshotting stores in f3e2abb3d075e2d6a5c2948af99b8c71 at 1732137263820Finished memstore snapshotting TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732137263821 (+1 ms)Flushing stores of TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71. at 1732137263822 (+1 ms)Flushing f3e2abb3d075e2d6a5c2948af99b8c71/A: creating writer at 1732137263822Flushing f3e2abb3d075e2d6a5c2948af99b8c71/A: appending metadata at 1732137263825 (+3 ms)Flushing f3e2abb3d075e2d6a5c2948af99b8c71/A: closing flushed file at 1732137263825Flushing f3e2abb3d075e2d6a5c2948af99b8c71/B: creating writer at 1732137263835 (+10 ms)Flushing f3e2abb3d075e2d6a5c2948af99b8c71/B: appending metadata at 1732137263838 (+3 ms)Flushing f3e2abb3d075e2d6a5c2948af99b8c71/B: closing flushed file at 1732137263838Flushing f3e2abb3d075e2d6a5c2948af99b8c71/C: creating writer at 1732137263851 (+13 ms)Flushing f3e2abb3d075e2d6a5c2948af99b8c71/C: appending metadata at 1732137263854 (+3 ms)Flushing f3e2abb3d075e2d6a5c2948af99b8c71/C: closing flushed file at 1732137263854Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71ab3984: reopening flushed file at 1732137263864 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51d6d81: reopening flushed file at 1732137263869 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b807519: reopening flushed file at 1732137263875 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for f3e2abb3d075e2d6a5c2948af99b8c71 in 62ms, sequenceid=690, compaction requested=true at 1732137263882 (+7 ms)Writing region close event to WAL at 1732137264044 (+162 ms)Running coprocessor post-close hooks at 1732137264048 (+4 ms)Closed at 1732137264048 2024-11-20T21:14:24,052 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:24,052 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=f3e2abb3d075e2d6a5c2948af99b8c71, regionState=CLOSED 2024-11-20T21:14:24,054 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 because future has completed 2024-11-20T21:14:24,057 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-20T21:14:24,057 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure f3e2abb3d075e2d6a5c2948af99b8c71, server=0b16ab3f13b7,36987,1732137236536 in 392 msec 2024-11-20T21:14:24,060 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-20T21:14:24,060 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f3e2abb3d075e2d6a5c2948af99b8c71, UNASSIGN in 399 msec 2024-11-20T21:14:24,062 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-20T21:14:24,063 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 408 msec 2024-11-20T21:14:24,064 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732137264064"}]},"ts":"1732137264064"} 2024-11-20T21:14:24,066 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T21:14:24,066 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T21:14:24,068 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 422 msec 2024-11-20T21:14:24,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-20T21:14:24,278 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:14:24,282 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-20T21:14:24,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:14:24,289 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:14:24,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-20T21:14:24,291 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:14:24,299 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:24,304 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A, FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B, FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C, FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/recovered.edits] 2024-11-20T21:14:24,308 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/075a874c37174bf4b2d6f7223c27a5ac to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/075a874c37174bf4b2d6f7223c27a5ac 2024-11-20T21:14:24,311 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/bccf272c2ac84ea790f2af9b72dabad9 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/bccf272c2ac84ea790f2af9b72dabad9 2024-11-20T21:14:24,317 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/dc430b6c48d644cd96a21c48a6043f80 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/dc430b6c48d644cd96a21c48a6043f80 2024-11-20T21:14:24,321 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/ead28636377c4a9483edf97de5ce18fe to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/A/ead28636377c4a9483edf97de5ce18fe 2024-11-20T21:14:24,325 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/0af203c9d16e41ae853a427d3b7774ad to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/0af203c9d16e41ae853a427d3b7774ad 2024-11-20T21:14:24,326 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/193b0e69e7204b9c88a863202f56f2dd to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/193b0e69e7204b9c88a863202f56f2dd 2024-11-20T21:14:24,328 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/3f90d07075604c869323d79b75a0e4f0 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/3f90d07075604c869323d79b75a0e4f0 2024-11-20T21:14:24,330 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/d23be0ddcfe74852855ac8f188268973 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/B/d23be0ddcfe74852855ac8f188268973 2024-11-20T21:14:24,334 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/168c757e3e8d44a0a0d966b683e1a51f to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/168c757e3e8d44a0a0d966b683e1a51f 2024-11-20T21:14:24,336 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/6b33a08618534549aad707ea408e47f4 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/6b33a08618534549aad707ea408e47f4 2024-11-20T21:14:24,338 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/83a7703c3cbf4fdcbbf5abd181287427 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/83a7703c3cbf4fdcbbf5abd181287427 2024-11-20T21:14:24,339 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/ad9a6a9750d24d5aa51b37ca8bf9738d to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/C/ad9a6a9750d24d5aa51b37ca8bf9738d 2024-11-20T21:14:24,344 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/recovered.edits/693.seqid to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71/recovered.edits/693.seqid 2024-11-20T21:14:24,344 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/f3e2abb3d075e2d6a5c2948af99b8c71 2024-11-20T21:14:24,345 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T21:14:24,352 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:14:24,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-20T21:14:24,365 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T21:14:24,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-20T21:14:24,406 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T21:14:24,407 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:14:24,408 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T21:14:24,408 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732137264408"}]},"ts":"9223372036854775807"} 2024-11-20T21:14:24,413 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T21:14:24,413 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f3e2abb3d075e2d6a5c2948af99b8c71, NAME => 'TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T21:14:24,413 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T21:14:24,413 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732137264413"}]},"ts":"9223372036854775807"} 2024-11-20T21:14:24,417 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T21:14:24,418 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:14:24,420 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 135 msec 2024-11-20T21:14:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-20T21:14:24,608 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T21:14:24,608 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:14:24,622 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 209) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;0b16ab3f13b7:36987-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_209861989_22 at /127.0.0.1:36190 [Waiting for operation #265] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1021389946_22 at /127.0.0.1:43174 [Waiting for operation #149] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1021389946_22 at /127.0.0.1:58636 [Waiting for operation #244] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=567 (was 515) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=858 (was 1643) 2024-11-20T21:14:24,640 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=567, ProcessCount=11, AvailableMemoryMB=849 2024-11-20T21:14:24,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T21:14:24,642 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T21:14:24,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T21:14:24,648 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T21:14:24,648 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:24,648 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-20T21:14:24,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-20T21:14:24,650 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T21:14:24,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741942_1118 (size=1061) 2024-11-20T21:14:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-20T21:14:24,814 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T21:14:24,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-20T21:14:25,077 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094 2024-11-20T21:14:25,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741943_1119 (size=53) 2024-11-20T21:14:25,084 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:14:25,084 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 917d9f6cd1adb0d5f4ed70d067a87108, disabling compactions & flushes 2024-11-20T21:14:25,084 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:25,084 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:25,084 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. after waiting 0 ms 2024-11-20T21:14:25,084 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:25,084 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:25,084 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 917d9f6cd1adb0d5f4ed70d067a87108: Waiting for close lock at 1732137265084Disabling compacts and flushes for region at 1732137265084Disabling writes for close at 1732137265084Writing region close event to WAL at 1732137265084Closed at 1732137265084 2024-11-20T21:14:25,086 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T21:14:25,086 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732137265086"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732137265086"}]},"ts":"1732137265086"} 2024-11-20T21:14:25,088 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T21:14:25,089 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T21:14:25,089 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732137265089"}]},"ts":"1732137265089"} 2024-11-20T21:14:25,091 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T21:14:25,091 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=917d9f6cd1adb0d5f4ed70d067a87108, ASSIGN}] 2024-11-20T21:14:25,092 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=917d9f6cd1adb0d5f4ed70d067a87108, ASSIGN 2024-11-20T21:14:25,093 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=917d9f6cd1adb0d5f4ed70d067a87108, ASSIGN; state=OFFLINE, location=0b16ab3f13b7,36987,1732137236536; forceNewPlan=false, retain=false 2024-11-20T21:14:25,244 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=917d9f6cd1adb0d5f4ed70d067a87108, regionState=OPENING, regionLocation=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:25,246 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=917d9f6cd1adb0d5f4ed70d067a87108, ASSIGN because future has completed 2024-11-20T21:14:25,247 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536}] 2024-11-20T21:14:25,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-20T21:14:25,403 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:25,403 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:14:25,404 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:25,404 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:14:25,404 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:25,404 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:25,406 INFO [StoreOpener-917d9f6cd1adb0d5f4ed70d067a87108-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:25,407 INFO [StoreOpener-917d9f6cd1adb0d5f4ed70d067a87108-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T21:14:25,408 INFO [StoreOpener-917d9f6cd1adb0d5f4ed70d067a87108-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 917d9f6cd1adb0d5f4ed70d067a87108 columnFamilyName A 2024-11-20T21:14:25,408 DEBUG [StoreOpener-917d9f6cd1adb0d5f4ed70d067a87108-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:25,408 INFO [StoreOpener-917d9f6cd1adb0d5f4ed70d067a87108-1 {}] regionserver.HStore(327): Store=917d9f6cd1adb0d5f4ed70d067a87108/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:14:25,408 INFO [StoreOpener-917d9f6cd1adb0d5f4ed70d067a87108-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:25,409 INFO [StoreOpener-917d9f6cd1adb0d5f4ed70d067a87108-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T21:14:25,410 INFO [StoreOpener-917d9f6cd1adb0d5f4ed70d067a87108-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 917d9f6cd1adb0d5f4ed70d067a87108 columnFamilyName B 2024-11-20T21:14:25,410 DEBUG [StoreOpener-917d9f6cd1adb0d5f4ed70d067a87108-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:25,410 INFO [StoreOpener-917d9f6cd1adb0d5f4ed70d067a87108-1 {}] regionserver.HStore(327): Store=917d9f6cd1adb0d5f4ed70d067a87108/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:14:25,410 INFO [StoreOpener-917d9f6cd1adb0d5f4ed70d067a87108-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:25,411 INFO [StoreOpener-917d9f6cd1adb0d5f4ed70d067a87108-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T21:14:25,411 INFO [StoreOpener-917d9f6cd1adb0d5f4ed70d067a87108-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 917d9f6cd1adb0d5f4ed70d067a87108 columnFamilyName C 2024-11-20T21:14:25,411 DEBUG [StoreOpener-917d9f6cd1adb0d5f4ed70d067a87108-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:25,412 INFO [StoreOpener-917d9f6cd1adb0d5f4ed70d067a87108-1 {}] regionserver.HStore(327): Store=917d9f6cd1adb0d5f4ed70d067a87108/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:14:25,412 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:25,412 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:25,412 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:25,413 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:25,414 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:25,414 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:25,414 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T21:14:25,415 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:25,417 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T21:14:25,417 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 917d9f6cd1adb0d5f4ed70d067a87108; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60522673, jitterRate=-0.09814189374446869}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T21:14:25,418 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:25,418 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 917d9f6cd1adb0d5f4ed70d067a87108: Running coprocessor pre-open hook at 1732137265404Writing region info on filesystem at 1732137265404Initializing all the Stores at 1732137265405 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732137265405Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732137265405Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732137265405Cleaning up temporary data from old regions at 1732137265414 (+9 ms)Running coprocessor post-open hooks at 1732137265418 (+4 ms)Region opened successfully at 1732137265418 2024-11-20T21:14:25,419 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., pid=34, masterSystemTime=1732137265400 2024-11-20T21:14:25,420 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:25,421 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:25,421 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=917d9f6cd1adb0d5f4ed70d067a87108, regionState=OPEN, openSeqNum=2, regionLocation=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:25,423 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 because future has completed 2024-11-20T21:14:25,425 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-20T21:14:25,425 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 in 177 msec 2024-11-20T21:14:25,427 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-20T21:14:25,427 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=917d9f6cd1adb0d5f4ed70d067a87108, ASSIGN in 334 msec 2024-11-20T21:14:25,427 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T21:14:25,428 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732137265427"}]},"ts":"1732137265427"} 2024-11-20T21:14:25,429 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T21:14:25,430 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T21:14:25,431 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 788 msec 2024-11-20T21:14:25,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-20T21:14:25,787 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:14:25,788 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49a00013, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:25,788 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:14:25,789 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:14:25,790 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:14:25,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:14:25,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:14:25,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13023e84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:25,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:14:25,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:14:25,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:25,792 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34118, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:14:25,793 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@329c9ead, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:25,793 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:14:25,794 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:14:25,795 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:14:25,796 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:14:25,798 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T21:14:25,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 0b16ab3f13b7,39687,1732137235812 2024-11-20T21:14:25,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1eb25f6a 2024-11-20T21:14:25,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T21:14:25,800 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T21:14:25,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T21:14:25,806 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T21:14:25,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-20T21:14:25,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741944_1120 (size=1097) 2024-11-20T21:14:26,229 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-20T21:14:26,230 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-20T21:14:26,235 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T21:14:26,241 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=917d9f6cd1adb0d5f4ed70d067a87108, REOPEN/MOVE}] 2024-11-20T21:14:26,242 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=917d9f6cd1adb0d5f4ed70d067a87108, REOPEN/MOVE 2024-11-20T21:14:26,243 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=917d9f6cd1adb0d5f4ed70d067a87108, regionState=CLOSING, regionLocation=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:26,244 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=917d9f6cd1adb0d5f4ed70d067a87108, REOPEN/MOVE because future has completed 2024-11-20T21:14:26,245 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T21:14:26,245 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536}] 2024-11-20T21:14:26,398 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:26,398 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T21:14:26,398 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 917d9f6cd1adb0d5f4ed70d067a87108, disabling compactions & flushes 2024-11-20T21:14:26,398 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:26,398 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:26,398 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. after waiting 0 ms 2024-11-20T21:14:26,398 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:26,402 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T21:14:26,403 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:26,403 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 917d9f6cd1adb0d5f4ed70d067a87108: Waiting for close lock at 1732137266398Running coprocessor pre-close hooks at 1732137266398Disabling compacts and flushes for region at 1732137266398Disabling writes for close at 1732137266398Writing region close event to WAL at 1732137266399 (+1 ms)Running coprocessor post-close hooks at 1732137266403 (+4 ms)Closed at 1732137266403 2024-11-20T21:14:26,403 WARN [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: 917d9f6cd1adb0d5f4ed70d067a87108 to self. 2024-11-20T21:14:26,405 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:26,405 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=917d9f6cd1adb0d5f4ed70d067a87108, regionState=CLOSED 2024-11-20T21:14:26,407 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 because future has completed 2024-11-20T21:14:26,410 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-20T21:14:26,410 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure 917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 in 163 msec 2024-11-20T21:14:26,411 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=917d9f6cd1adb0d5f4ed70d067a87108, REOPEN/MOVE; state=CLOSED, location=0b16ab3f13b7,36987,1732137236536; forceNewPlan=false, retain=true 2024-11-20T21:14:26,561 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=917d9f6cd1adb0d5f4ed70d067a87108, regionState=OPENING, regionLocation=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:26,563 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=917d9f6cd1adb0d5f4ed70d067a87108, REOPEN/MOVE because future has completed 2024-11-20T21:14:26,563 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536}] 2024-11-20T21:14:26,718 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:26,718 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:14:26,718 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:26,719 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:14:26,719 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:26,719 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:26,723 INFO [StoreOpener-917d9f6cd1adb0d5f4ed70d067a87108-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:26,724 INFO [StoreOpener-917d9f6cd1adb0d5f4ed70d067a87108-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T21:14:26,729 INFO [StoreOpener-917d9f6cd1adb0d5f4ed70d067a87108-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 917d9f6cd1adb0d5f4ed70d067a87108 columnFamilyName A 2024-11-20T21:14:26,731 DEBUG [StoreOpener-917d9f6cd1adb0d5f4ed70d067a87108-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:26,731 INFO [StoreOpener-917d9f6cd1adb0d5f4ed70d067a87108-1 {}] regionserver.HStore(327): Store=917d9f6cd1adb0d5f4ed70d067a87108/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:14:26,732 INFO [StoreOpener-917d9f6cd1adb0d5f4ed70d067a87108-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:26,732 INFO [StoreOpener-917d9f6cd1adb0d5f4ed70d067a87108-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T21:14:26,733 INFO [StoreOpener-917d9f6cd1adb0d5f4ed70d067a87108-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 917d9f6cd1adb0d5f4ed70d067a87108 columnFamilyName B 2024-11-20T21:14:26,733 DEBUG [StoreOpener-917d9f6cd1adb0d5f4ed70d067a87108-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:26,733 INFO [StoreOpener-917d9f6cd1adb0d5f4ed70d067a87108-1 {}] regionserver.HStore(327): Store=917d9f6cd1adb0d5f4ed70d067a87108/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:14:26,733 INFO [StoreOpener-917d9f6cd1adb0d5f4ed70d067a87108-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:26,734 INFO [StoreOpener-917d9f6cd1adb0d5f4ed70d067a87108-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T21:14:26,734 INFO [StoreOpener-917d9f6cd1adb0d5f4ed70d067a87108-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 917d9f6cd1adb0d5f4ed70d067a87108 columnFamilyName C 2024-11-20T21:14:26,734 DEBUG [StoreOpener-917d9f6cd1adb0d5f4ed70d067a87108-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:26,735 INFO [StoreOpener-917d9f6cd1adb0d5f4ed70d067a87108-1 {}] regionserver.HStore(327): Store=917d9f6cd1adb0d5f4ed70d067a87108/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:14:26,735 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:26,735 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:26,736 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:26,737 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:26,738 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:26,738 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:26,739 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T21:14:26,740 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:26,741 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 917d9f6cd1adb0d5f4ed70d067a87108; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63567417, jitterRate=-0.0527716726064682}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T21:14:26,741 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:26,743 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 917d9f6cd1adb0d5f4ed70d067a87108: Running coprocessor pre-open hook at 1732137266719Writing region info on filesystem at 1732137266719Initializing all the Stores at 1732137266720 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732137266720Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732137266720Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732137266720Cleaning up temporary data from old regions at 1732137266738 (+18 ms)Running coprocessor post-open hooks at 1732137266741 (+3 ms)Region opened successfully at 1732137266743 (+2 ms) 2024-11-20T21:14:26,744 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., pid=39, masterSystemTime=1732137266715 2024-11-20T21:14:26,746 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:26,746 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:26,747 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=917d9f6cd1adb0d5f4ed70d067a87108, regionState=OPEN, openSeqNum=5, regionLocation=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:26,750 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 because future has completed 2024-11-20T21:14:26,753 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-20T21:14:26,753 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 in 188 msec 2024-11-20T21:14:26,754 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-20T21:14:26,755 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=917d9f6cd1adb0d5f4ed70d067a87108, REOPEN/MOVE in 512 msec 2024-11-20T21:14:26,758 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-20T21:14:26,758 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 522 msec 2024-11-20T21:14:26,763 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 950 msec 2024-11-20T21:14:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-20T21:14:26,765 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:14:26,766 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ad8b864, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:26,766 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:14:26,766 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:14:26,767 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:14:26,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:14:26,767 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:14:26,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f4de448, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:26,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:14:26,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:14:26,768 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:26,769 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34158, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:14:26,770 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61e16802, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:26,773 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5adc4e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:26,773 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:14:26,773 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:14:26,774 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:14:26,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:14:26,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:14:26,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13e246ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:26,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:14:26,774 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:14:26,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:26,776 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:14:26,776 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7915ba55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:26,777 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681c9b14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:26,777 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:14:26,777 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:14:26,778 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:14:26,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:14:26,778 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:14:26,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@175807c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:26,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:14:26,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:14:26,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:26,780 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34196, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:14:26,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@742fcf5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:26,788 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d0a90e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:26,788 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:14:26,788 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:14:26,789 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:14:26,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:14:26,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:14:26,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f1e551c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:26,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:14:26,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:14:26,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:26,790 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:14:26,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@195293ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:26,791 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771f5e02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:26,791 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:14:26,791 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:14:26,792 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:14:26,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:14:26,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:14:26,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7074cd9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:26,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:14:26,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:14:26,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:26,794 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:14:26,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@650d4324, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:26,795 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ffb7a39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:26,795 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:14:26,796 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:14:26,796 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:14:26,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:14:26,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:14:26,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6617779a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:26,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:14:26,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:14:26,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:26,798 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34234, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:14:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77de28ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:26,801 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5453a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:26,801 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:14:26,801 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:14:26,802 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:14:26,802 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:14:26,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:14:26,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@127742ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:26,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:14:26,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:14:26,803 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:26,804 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34254, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:14:26,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29db9dab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:26,805 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73b6e690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:26,805 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:14:26,805 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:14:26,806 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:14:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:14:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:14:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ef73b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:14:26,806 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:14:26,807 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:26,807 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:14:26,808 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c0eac94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:26,809 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bd05ea8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:26,809 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:14:26,809 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:14:26,809 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:14:26,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:14:26,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:14:26,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25b91fc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:26,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:14:26,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:14:26,810 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:26,811 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:14:26,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a6df162, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:14:26,812 DEBUG [Thread-609 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:14:26,812 DEBUG [Thread-615 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:14:26,812 DEBUG [Thread-617 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:14:26,812 DEBUG [Thread-620 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:14:26,813 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:14:26,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:14:26,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:14:26,814 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:14:26,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:14:26,814 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:14:26,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:14:26,815 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41238, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:14:26,815 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41242, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:14:26,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-20T21:14:26,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-20T21:14:26,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:14:26,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:14:26,818 DEBUG [Thread-624 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:14:26,818 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41252, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:14:26,818 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41268, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:14:26,818 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:14:26,819 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:14:26,819 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:14:26,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5] 2024-11-20T21:14:26,820 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:14:26,820 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:14:26,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5] 2024-11-20T21:14:26,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5] 2024-11-20T21:14:26,826 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41282, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:14:26,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5] 2024-11-20T21:14:26,829 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5] 2024-11-20T21:14:26,836 DEBUG [Thread-622 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:14:26,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:14:26,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:14:26,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:26,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 917d9f6cd1adb0d5f4ed70d067a87108 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T21:14:26,843 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:14:26,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5] 2024-11-20T21:14:26,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=A 2024-11-20T21:14:26,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:26,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=B 2024-11-20T21:14:26,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:26,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=C 2024-11-20T21:14:26,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:26,849 DEBUG [Thread-611 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:14:26,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:14:26,850 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:14:26,852 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:14:26,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5] 2024-11-20T21:14:26,860 DEBUG [Thread-613 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:14:26,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:14:26,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:14:26,863 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:14:26,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5] 2024-11-20T21:14:26,871 DEBUG [Thread-626 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:14:26,872 DEBUG [Thread-626 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:14:26,872 DEBUG [Thread-626 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:14:26,875 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:14:26,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5] 2024-11-20T21:14:26,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1732137266851, max timestamp=1732137266919Num uniques 859; 2024-11-20T21:14:26,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-20T21:14:26,926 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732137266893, max timestamp=1732137266919Num uniques 859; 2024-11-20T21:14:26,926 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T21:14:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-20T21:14:26,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732137266893, max timestamp=1732137266921Num uniques 859; 2024-11-20T21:14:26,929 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T21:14:26,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209ad8eddf2f5e4dca8de8531396273f4c_917d9f6cd1adb0d5f4ed70d067a87108 is 50, key is test_row_0/A:col10/1732137266839/Put/seqid=0 2024-11-20T21:14:26,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T21:14:26,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:26,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:26,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:26,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:26,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:26,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:26,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137326973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:26,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:26,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:26,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:26,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741945_1121 (size=12154) 2024-11-20T21:14:26,993 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:26,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137326986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:26,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:26,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:26,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:26,996 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137326986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:26,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:26,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:26,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:26,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137326986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:26,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:26,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:26,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:26,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137326987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:26,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:26,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:26,997 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:26,998 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209ad8eddf2f5e4dca8de8531396273f4c_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209ad8eddf2f5e4dca8de8531396273f4c_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:26,999 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/4c7ae6631a9f46c7b1c506f50f359276, store: [table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:27,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/4c7ae6631a9f46c7b1c506f50f359276 is 175, key is test_row_0/A:col10/1732137266839/Put/seqid=0 2024-11-20T21:14:27,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741946_1122 (size=30955) 2024-11-20T21:14:27,043 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/4c7ae6631a9f46c7b1c506f50f359276 2024-11-20T21:14:27,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:27,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137327103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:27,115 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/3eb734e966674494902636e819cd6db9 is 50, key is test_row_0/B:col10/1732137266839/Put/seqid=0 2024-11-20T21:14:27,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:27,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:27,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137327109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:27,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:27,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137327109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137327110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:27,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:27,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137327109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:27,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:27,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T21:14:27,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:27,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:27,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:27,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:27,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-20T21:14:27,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741947_1123 (size=12001) 2024-11-20T21:14:27,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/3eb734e966674494902636e819cd6db9 2024-11-20T21:14:27,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/f002de1bec3343b2830bb54e392cbb21 is 50, key is test_row_0/C:col10/1732137266839/Put/seqid=0 2024-11-20T21:14:27,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741948_1124 (size=12001) 2024-11-20T21:14:27,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/f002de1bec3343b2830bb54e392cbb21 2024-11-20T21:14:27,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/4c7ae6631a9f46c7b1c506f50f359276 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/4c7ae6631a9f46c7b1c506f50f359276 2024-11-20T21:14:27,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T21:14:27,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:27,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:27,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:27,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:27,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:27,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/4c7ae6631a9f46c7b1c506f50f359276, entries=150, sequenceid=15, filesize=30.2 K 2024-11-20T21:14:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:27,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/3eb734e966674494902636e819cd6db9 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/3eb734e966674494902636e819cd6db9 2024-11-20T21:14:27,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/3eb734e966674494902636e819cd6db9, entries=150, sequenceid=15, filesize=11.7 K 2024-11-20T21:14:27,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/f002de1bec3343b2830bb54e392cbb21 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/f002de1bec3343b2830bb54e392cbb21 2024-11-20T21:14:27,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/f002de1bec3343b2830bb54e392cbb21, entries=150, sequenceid=15, filesize=11.7 K 2024-11-20T21:14:27,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 917d9f6cd1adb0d5f4ed70d067a87108 in 485ms, sequenceid=15, compaction requested=false 2024-11-20T21:14:27,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:27,340 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 917d9f6cd1adb0d5f4ed70d067a87108 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-20T21:14:27,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=A 2024-11-20T21:14:27,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:14:27,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=B 2024-11-20T21:14:27,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:14:27,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=C 2024-11-20T21:14:27,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:14:27,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:27,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137327371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:27,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:27,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:27,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137327376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:27,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:27,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:27,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137327383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:27,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:27,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137327385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:27,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:27,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:27,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:27,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137327396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:27,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:27,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120800a69997f524edbb13e7b8b4cc5f456_917d9f6cd1adb0d5f4ed70d067a87108 is 50, key is test_row_0/A:col10/1732137267332/Put/seqid=0 2024-11-20T21:14:27,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-20T21:14:27,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T21:14:27,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:27,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:27,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:27,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:27,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:27,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:27,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741949_1125 (size=12154) 2024-11-20T21:14:27,486 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:27,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:27,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137327508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:27,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:27,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137327508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:27,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:27,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137327508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:27,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:27,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:27,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:27,519 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120800a69997f524edbb13e7b8b4cc5f456_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120800a69997f524edbb13e7b8b4cc5f456_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137327509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:27,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,525 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:27,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137327517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:27,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,528 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:27,532 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/0b9ca9e5774a4faba145c0c9fa598068, store: [table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:27,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/0b9ca9e5774a4faba145c0c9fa598068 is 175, key is test_row_0/A:col10/1732137267332/Put/seqid=0 2024-11-20T21:14:27,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741950_1126 (size=30955) 2024-11-20T21:14:27,546 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/0b9ca9e5774a4faba145c0c9fa598068 2024-11-20T21:14:27,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/bf8fb2f64c744f2d8a2f71559ec37d64 is 50, key is test_row_0/B:col10/1732137267332/Put/seqid=0 2024-11-20T21:14:27,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741951_1127 (size=12001) 2024-11-20T21:14:27,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/bf8fb2f64c744f2d8a2f71559ec37d64 2024-11-20T21:14:27,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/4cf4f1e16383444dbeeeee6d728e0e74 is 50, key is test_row_0/C:col10/1732137267332/Put/seqid=0 2024-11-20T21:14:27,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741952_1128 (size=12001) 2024-11-20T21:14:27,596 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/4cf4f1e16383444dbeeeee6d728e0e74 2024-11-20T21:14:27,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/0b9ca9e5774a4faba145c0c9fa598068 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/0b9ca9e5774a4faba145c0c9fa598068 2024-11-20T21:14:27,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T21:14:27,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:27,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:27,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:27,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:27,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:27,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/0b9ca9e5774a4faba145c0c9fa598068, entries=150, sequenceid=52, filesize=30.2 K 2024-11-20T21:14:27,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/bf8fb2f64c744f2d8a2f71559ec37d64 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/bf8fb2f64c744f2d8a2f71559ec37d64 2024-11-20T21:14:27,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/bf8fb2f64c744f2d8a2f71559ec37d64, entries=150, sequenceid=52, filesize=11.7 K 2024-11-20T21:14:27,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/4cf4f1e16383444dbeeeee6d728e0e74 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/4cf4f1e16383444dbeeeee6d728e0e74 2024-11-20T21:14:27,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/4cf4f1e16383444dbeeeee6d728e0e74, entries=150, sequenceid=52, filesize=11.7 K 2024-11-20T21:14:27,638 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 917d9f6cd1adb0d5f4ed70d067a87108 in 298ms, sequenceid=52, compaction requested=false 2024-11-20T21:14:27,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:27,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:27,733 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 917d9f6cd1adb0d5f4ed70d067a87108 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-20T21:14:27,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=A 2024-11-20T21:14:27,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:27,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=B 2024-11-20T21:14:27,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:27,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=C 2024-11-20T21:14:27,733 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:27,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202947acca4c19456097a6b64bad8f28ec_917d9f6cd1adb0d5f4ed70d067a87108 is 50, key is test_row_0/A:col10/1732137267730/Put/seqid=0 2024-11-20T21:14:27,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T21:14:27,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:27,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:27,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:27,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:27,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:27,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:27,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741953_1129 (size=14594) 2024-11-20T21:14:27,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:27,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137327786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:27,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:27,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137327786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:27,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:27,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137327787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:27,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137327788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,796 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:27,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:27,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137327794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:27,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:27,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137327897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:27,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:27,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137327898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:27,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,901 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:27,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:27,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137327898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:27,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:27,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:27,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:27,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137327907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:27,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:27,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137327908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:27,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:27,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:27,914 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:27,918 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T21:14:27,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:27,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:27,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:27,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:27,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:27,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-20T21:14:28,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T21:14:28,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:28,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:28,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:28,077 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:28,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:28,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:28,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:28,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137328107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:28,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137328107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:28,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:28,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137328108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:28,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:28,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,115 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:28,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,115 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:28,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137328117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:28,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:28,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137328117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:28,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:28,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:28,174 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:28,222 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202947acca4c19456097a6b64bad8f28ec_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202947acca4c19456097a6b64bad8f28ec_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:28,224 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/302735d4e24c4ffaa00f5bcd99557e8b, store: [table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:28,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/302735d4e24c4ffaa00f5bcd99557e8b is 175, key is test_row_0/A:col10/1732137267730/Put/seqid=0 2024-11-20T21:14:28,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T21:14:28,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:28,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:28,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:28,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:28,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:28,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:28,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741954_1130 (size=39549) 2024-11-20T21:14:28,259 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=65, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/302735d4e24c4ffaa00f5bcd99557e8b 2024-11-20T21:14:28,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/887fcf60681e4fb78cd9cf4dc9aafb58 is 50, key is test_row_0/B:col10/1732137267730/Put/seqid=0 2024-11-20T21:14:28,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741955_1131 (size=12001) 2024-11-20T21:14:28,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/887fcf60681e4fb78cd9cf4dc9aafb58 2024-11-20T21:14:28,352 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/78af7d6edcc24fbd8503964a541cddfc is 50, key is test_row_0/C:col10/1732137267730/Put/seqid=0 2024-11-20T21:14:28,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741956_1132 (size=12001) 2024-11-20T21:14:28,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=65 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/78af7d6edcc24fbd8503964a541cddfc 2024-11-20T21:14:28,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/302735d4e24c4ffaa00f5bcd99557e8b as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/302735d4e24c4ffaa00f5bcd99557e8b 2024-11-20T21:14:28,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/302735d4e24c4ffaa00f5bcd99557e8b, entries=200, sequenceid=65, filesize=38.6 K 2024-11-20T21:14:28,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/887fcf60681e4fb78cd9cf4dc9aafb58 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/887fcf60681e4fb78cd9cf4dc9aafb58 2024-11-20T21:14:28,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T21:14:28,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:28,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:28,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:28,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:28,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:28,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:28,400 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/887fcf60681e4fb78cd9cf4dc9aafb58, entries=150, sequenceid=65, filesize=11.7 K 2024-11-20T21:14:28,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/78af7d6edcc24fbd8503964a541cddfc as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/78af7d6edcc24fbd8503964a541cddfc 2024-11-20T21:14:28,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/78af7d6edcc24fbd8503964a541cddfc, entries=150, sequenceid=65, filesize=11.7 K 2024-11-20T21:14:28,414 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 917d9f6cd1adb0d5f4ed70d067a87108 in 682ms, sequenceid=65, compaction requested=true 2024-11-20T21:14:28,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:28,415 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:28,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:14:28,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:28,415 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:28,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:14:28,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:28,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:14:28,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:28,416 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:28,416 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 917d9f6cd1adb0d5f4ed70d067a87108/A is initiating minor compaction (all files) 2024-11-20T21:14:28,416 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 917d9f6cd1adb0d5f4ed70d067a87108/A in TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:28,416 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/4c7ae6631a9f46c7b1c506f50f359276, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/0b9ca9e5774a4faba145c0c9fa598068, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/302735d4e24c4ffaa00f5bcd99557e8b] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp, totalSize=99.1 K 2024-11-20T21:14:28,416 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:28,417 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 917d9f6cd1adb0d5f4ed70d067a87108/B is initiating minor compaction (all files) 2024-11-20T21:14:28,417 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 917d9f6cd1adb0d5f4ed70d067a87108/B in TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:28,417 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/3eb734e966674494902636e819cd6db9, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/bf8fb2f64c744f2d8a2f71559ec37d64, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/887fcf60681e4fb78cd9cf4dc9aafb58] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp, totalSize=35.2 K 2024-11-20T21:14:28,416 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:28,417 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. files: [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/4c7ae6631a9f46c7b1c506f50f359276, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/0b9ca9e5774a4faba145c0c9fa598068, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/302735d4e24c4ffaa00f5bcd99557e8b] 2024-11-20T21:14:28,417 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 3eb734e966674494902636e819cd6db9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732137266835 2024-11-20T21:14:28,418 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c7ae6631a9f46c7b1c506f50f359276, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732137266835 2024-11-20T21:14:28,418 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting bf8fb2f64c744f2d8a2f71559ec37d64, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732137266971 2024-11-20T21:14:28,418 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b9ca9e5774a4faba145c0c9fa598068, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732137266971 2024-11-20T21:14:28,419 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 887fcf60681e4fb78cd9cf4dc9aafb58, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1732137267720 2024-11-20T21:14:28,419 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 302735d4e24c4ffaa00f5bcd99557e8b, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1732137267720 2024-11-20T21:14:28,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:28,433 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 917d9f6cd1adb0d5f4ed70d067a87108 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T21:14:28,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=A 2024-11-20T21:14:28,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:28,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=B 2024-11-20T21:14:28,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:28,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=C 2024-11-20T21:14:28,434 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:28,446 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 917d9f6cd1adb0d5f4ed70d067a87108#B#compaction#114 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:28,446 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/d3085ff7756847a9b4faaa620ce3725c is 50, key is test_row_0/B:col10/1732137267730/Put/seqid=0 2024-11-20T21:14:28,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:28,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137328449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:28,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:28,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:28,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137328450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:28,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:28,457 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:28,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:28,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137328453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:28,459 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:28,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:28,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137328454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:28,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:28,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:28,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137328455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:28,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:28,507 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120559fa15633cd47e9b34d36ffb2b3d640_917d9f6cd1adb0d5f4ed70d067a87108 store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:28,512 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120559fa15633cd47e9b34d36ffb2b3d640_917d9f6cd1adb0d5f4ed70d067a87108, store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:28,516 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120559fa15633cd47e9b34d36ffb2b3d640_917d9f6cd1adb0d5f4ed70d067a87108 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:28,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741957_1133 (size=12104) 2024-11-20T21:14:28,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T21:14:28,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:28,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:28,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:28,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:28,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:28,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:28,552 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/d3085ff7756847a9b4faaa620ce3725c as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/d3085ff7756847a9b4faaa620ce3725c 2024-11-20T21:14:28,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411204654345419db4fd3b2625ea95381bae0_917d9f6cd1adb0d5f4ed70d067a87108 is 50, key is test_row_0/A:col10/1732137268431/Put/seqid=0 2024-11-20T21:14:28,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:28,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137328569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:28,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:28,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137328570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:28,577 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:28,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:28,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137328570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137328570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:28,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:28,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137328570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:28,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:28,578 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 917d9f6cd1adb0d5f4ed70d067a87108/B of 917d9f6cd1adb0d5f4ed70d067a87108 into d3085ff7756847a9b4faaa620ce3725c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:28,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,578 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:28,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,578 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:28,578 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., storeName=917d9f6cd1adb0d5f4ed70d067a87108/B, priority=13, startTime=1732137268415; duration=0sec 2024-11-20T21:14:28,579 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:28,579 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:28,579 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:B 2024-11-20T21:14:28,579 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:28,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:28,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741958_1134 (size=4469) 2024-11-20T21:14:28,581 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:28,581 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 917d9f6cd1adb0d5f4ed70d067a87108/C is initiating minor compaction (all files) 2024-11-20T21:14:28,581 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 917d9f6cd1adb0d5f4ed70d067a87108/C in TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:28,581 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/f002de1bec3343b2830bb54e392cbb21, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/4cf4f1e16383444dbeeeee6d728e0e74, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/78af7d6edcc24fbd8503964a541cddfc] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp, totalSize=35.2 K 2024-11-20T21:14:28,582 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting f002de1bec3343b2830bb54e392cbb21, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732137266835 2024-11-20T21:14:28,582 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cf4f1e16383444dbeeeee6d728e0e74, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732137266971 2024-11-20T21:14:28,583 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 78af7d6edcc24fbd8503964a541cddfc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1732137267720 2024-11-20T21:14:28,586 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 917d9f6cd1adb0d5f4ed70d067a87108#A#compaction#115 average throughput is 0.19 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:28,587 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/f60d019128074d2a9efd4b3c22413da0 is 175, key is test_row_0/A:col10/1732137267730/Put/seqid=0 2024-11-20T21:14:28,631 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 917d9f6cd1adb0d5f4ed70d067a87108#C#compaction#117 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:28,632 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/3e517de421c64dd8a7f06e5ab417af16 is 50, key is test_row_0/C:col10/1732137267730/Put/seqid=0 2024-11-20T21:14:28,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741960_1136 (size=31058) 2024-11-20T21:14:28,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741959_1135 (size=14594) 2024-11-20T21:14:28,672 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:28,684 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/f60d019128074d2a9efd4b3c22413da0 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/f60d019128074d2a9efd4b3c22413da0 2024-11-20T21:14:28,694 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411204654345419db4fd3b2625ea95381bae0_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411204654345419db4fd3b2625ea95381bae0_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:28,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741961_1137 (size=12104) 2024-11-20T21:14:28,698 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/036c165b18f64c5fb1bd931fd43c5860, store: [table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:28,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/036c165b18f64c5fb1bd931fd43c5860 is 175, key is test_row_0/A:col10/1732137268431/Put/seqid=0 2024-11-20T21:14:28,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T21:14:28,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:28,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:28,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:28,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:28,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:28,701 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 917d9f6cd1adb0d5f4ed70d067a87108/A of 917d9f6cd1adb0d5f4ed70d067a87108 into f60d019128074d2a9efd4b3c22413da0(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:28,701 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:28,701 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., storeName=917d9f6cd1adb0d5f4ed70d067a87108/A, priority=13, startTime=1732137268414; duration=0sec 2024-11-20T21:14:28,702 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:28,702 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:A 2024-11-20T21:14:28,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:28,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741962_1138 (size=39549) 2024-11-20T21:14:28,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137328787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:28,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,794 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:28,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:28,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137328787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:28,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:28,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,795 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:28,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137328787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:28,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:28,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137328788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:28,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137328788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:28,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:28,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:28,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:28,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T21:14:28,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:28,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:28,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:28,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:28,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:28,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:28,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-20T21:14:29,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T21:14:29,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:29,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:29,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:29,011 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:29,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:29,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:29,104 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/3e517de421c64dd8a7f06e5ab417af16 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/3e517de421c64dd8a7f06e5ab417af16 2024-11-20T21:14:29,110 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 917d9f6cd1adb0d5f4ed70d067a87108/C of 917d9f6cd1adb0d5f4ed70d067a87108 into 3e517de421c64dd8a7f06e5ab417af16(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:29,110 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:29,110 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., storeName=917d9f6cd1adb0d5f4ed70d067a87108/C, priority=13, startTime=1732137268415; duration=0sec 2024-11-20T21:14:29,110 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:29,110 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:C 2024-11-20T21:14:29,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137329107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:29,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:29,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:29,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:29,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:29,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137329107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137329107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:29,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:29,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:29,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137329108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:29,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137329108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:29,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:29,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:29,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:29,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:29,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:29,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:29,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:29,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:29,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:29,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:29,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T21:14:29,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:29,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:29,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:29,164 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:29,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:29,170 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=85, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/036c165b18f64c5fb1bd931fd43c5860 2024-11-20T21:14:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:29,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/5006763660844be9be657840edb975dc is 50, key is test_row_0/B:col10/1732137268431/Put/seqid=0 2024-11-20T21:14:29,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741963_1139 (size=12001) 2024-11-20T21:14:29,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/5006763660844be9be657840edb975dc 2024-11-20T21:14:29,214 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/fce52246e3164f9d8c6b7aa16edb5861 is 50, key is test_row_0/C:col10/1732137268431/Put/seqid=0 2024-11-20T21:14:29,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741964_1140 (size=12001) 2024-11-20T21:14:29,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T21:14:29,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:29,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:29,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:29,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:29,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:29,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T21:14:29,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:29,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:29,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:29,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:29,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:29,579 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T21:14:29,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=85 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/fce52246e3164f9d8c6b7aa16edb5861 2024-11-20T21:14:29,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:29,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/036c165b18f64c5fb1bd931fd43c5860 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/036c165b18f64c5fb1bd931fd43c5860 2024-11-20T21:14:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137329627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:29,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:29,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:29,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:29,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137329628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:29,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:29,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:29,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137329628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137329628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:29,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:29,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:29,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:29,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:29,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:29,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:29,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137329628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:29,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:29,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:29,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:29,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:29,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:29,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:29,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:29,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T21:14:29,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:29,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:29,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:29,638 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:29,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:29,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:29,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/036c165b18f64c5fb1bd931fd43c5860, entries=200, sequenceid=85, filesize=38.6 K 2024-11-20T21:14:29,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/5006763660844be9be657840edb975dc as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/5006763660844be9be657840edb975dc 2024-11-20T21:14:29,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/5006763660844be9be657840edb975dc, entries=150, sequenceid=85, filesize=11.7 K 2024-11-20T21:14:29,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/fce52246e3164f9d8c6b7aa16edb5861 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/fce52246e3164f9d8c6b7aa16edb5861 2024-11-20T21:14:29,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/fce52246e3164f9d8c6b7aa16edb5861, entries=150, sequenceid=85, filesize=11.7 K 2024-11-20T21:14:29,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 917d9f6cd1adb0d5f4ed70d067a87108 in 1222ms, sequenceid=85, compaction requested=false 2024-11-20T21:14:29,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:29,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-20T21:14:29,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:29,792 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 917d9f6cd1adb0d5f4ed70d067a87108 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T21:14:29,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=A 2024-11-20T21:14:29,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:29,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=B 2024-11-20T21:14:29,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:29,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=C 2024-11-20T21:14:29,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:29,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120aae157af444c46fe9d71e56ada58a5fe_917d9f6cd1adb0d5f4ed70d067a87108 is 50, key is test_row_0/A:col10/1732137268451/Put/seqid=0 2024-11-20T21:14:29,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741965_1141 (size=12154) 2024-11-20T21:14:29,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:29,892 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120aae157af444c46fe9d71e56ada58a5fe_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120aae157af444c46fe9d71e56ada58a5fe_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:29,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/59951e166bff4857ab64443c3126d9dd, store: [table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:29,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/59951e166bff4857ab64443c3126d9dd is 175, key is test_row_0/A:col10/1732137268451/Put/seqid=0 2024-11-20T21:14:29,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741966_1142 (size=30955) 2024-11-20T21:14:29,944 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=100, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/59951e166bff4857ab64443c3126d9dd 2024-11-20T21:14:29,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/3bb27a91313842f6808c63336c55a976 is 50, key is test_row_0/B:col10/1732137268451/Put/seqid=0 2024-11-20T21:14:30,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741967_1143 (size=12001) 2024-11-20T21:14:30,409 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/3bb27a91313842f6808c63336c55a976 2024-11-20T21:14:30,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/54f37ed9bafb47e4be65153c408e371f is 50, key is test_row_0/C:col10/1732137268451/Put/seqid=0 2024-11-20T21:14:30,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741968_1144 (size=12001) 2024-11-20T21:14:30,457 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/54f37ed9bafb47e4be65153c408e371f 2024-11-20T21:14:30,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/59951e166bff4857ab64443c3126d9dd as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/59951e166bff4857ab64443c3126d9dd 2024-11-20T21:14:30,491 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/59951e166bff4857ab64443c3126d9dd, entries=150, sequenceid=100, filesize=30.2 K 2024-11-20T21:14:30,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/3bb27a91313842f6808c63336c55a976 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/3bb27a91313842f6808c63336c55a976 2024-11-20T21:14:30,507 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/3bb27a91313842f6808c63336c55a976, entries=150, sequenceid=100, filesize=11.7 K 2024-11-20T21:14:30,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/54f37ed9bafb47e4be65153c408e371f as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/54f37ed9bafb47e4be65153c408e371f 2024-11-20T21:14:30,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,525 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/54f37ed9bafb47e4be65153c408e371f, entries=150, sequenceid=100, filesize=11.7 K 2024-11-20T21:14:30,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,532 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for 917d9f6cd1adb0d5f4ed70d067a87108 in 740ms, sequenceid=100, compaction requested=true 2024-11-20T21:14:30,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:30,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:30,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-20T21:14:30,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-20T21:14:30,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,536 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-20T21:14:30,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,537 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.7140 sec 2024-11-20T21:14:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,539 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 3.7230 sec 2024-11-20T21:14:30,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:30,717 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 917d9f6cd1adb0d5f4ed70d067a87108 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T21:14:30,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=A 2024-11-20T21:14:30,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:30,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=B 2024-11-20T21:14:30,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:30,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=C 2024-11-20T21:14:30,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:30,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c8515c10512b4890a1c494e9d702092d_917d9f6cd1adb0d5f4ed70d067a87108 is 50, key is test_row_0/A:col10/1732137270696/Put/seqid=0 2024-11-20T21:14:30,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741969_1145 (size=21918) 2024-11-20T21:14:30,794 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:30,800 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c8515c10512b4890a1c494e9d702092d_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c8515c10512b4890a1c494e9d702092d_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:30,802 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/ed2f9165bed84c5b8b4ba260d50ffbcc, store: [table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:30,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/ed2f9165bed84c5b8b4ba260d50ffbcc is 175, key is test_row_0/A:col10/1732137270696/Put/seqid=0 2024-11-20T21:14:30,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732137270774, max timestamp=1732137270803Num uniques 859; 2024-11-20T21:14:30,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732137270774, max timestamp=1732137270803Num uniques 859; 2024-11-20T21:14:30,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T21:14:30,807 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T21:14:30,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732137270774, max timestamp=1732137270803Num uniques 859; 2024-11-20T21:14:30,813 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T21:14:30,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741970_1146 (size=65323) 2024-11-20T21:14:30,828 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=110, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/ed2f9165bed84c5b8b4ba260d50ffbcc 2024-11-20T21:14:30,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/238cd38c7a4f4def86e2685b8503be52 is 50, key is test_row_0/B:col10/1732137270696/Put/seqid=0 2024-11-20T21:14:30,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741971_1147 (size=12001) 2024-11-20T21:14:30,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:30,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137330930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:30,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:30,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137330933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:30,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:30,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:30,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:30,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:30,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:30,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137330935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:30,945 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:30,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:30,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:30,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:30,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:30,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137330945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:30,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137330946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:30,956 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:30,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:30,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:30,961 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:30,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-20T21:14:30,988 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:14:30,989 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:14:30,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-20T21:14:30,992 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:14:30,993 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:14:30,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:14:30,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T21:14:31,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:31,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137331047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:31,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:31,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:31,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:31,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:31,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137331057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:31,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:31,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:31,063 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:31,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:31,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137331058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:31,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:31,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:31,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:31,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:31,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137331067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:31,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:31,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:31,074 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:31,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137331067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:31,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:31,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:31,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:31,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T21:14:31,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T21:14:31,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:31,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:31,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:31,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:31,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:31,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:31,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:31,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137331260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:31,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:31,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:31,277 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:31,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:31,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137331266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:31,278 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:31,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:31,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137331277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:31,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:31,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:31,286 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:31,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:31,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137331281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:31,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:31,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:31,286 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:31,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/238cd38c7a4f4def86e2685b8503be52 2024-11-20T21:14:31,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:31,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137331287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:31,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:31,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:31,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:31,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T21:14:31,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:31,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:31,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:31,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:31,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:31,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T21:14:31,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:31,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/cc4f7501892841ca95fa2b8b6b4e94cd is 50, key is test_row_0/C:col10/1732137270696/Put/seqid=0 2024-11-20T21:14:31,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741972_1148 (size=12001) 2024-11-20T21:14:31,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T21:14:31,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:31,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:31,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:31,463 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:31,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:31,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:31,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:31,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137331588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:31,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:31,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:31,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:31,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137331588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:31,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:31,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:31,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:31,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137331603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:31,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:31,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137331603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:31,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:31,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:31,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137331608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:31,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:31,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:31,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:31,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:31,615 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:31,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:31,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:31,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:31,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T21:14:31,627 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T21:14:31,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:31,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:31,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:31,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:31,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:31,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T21:14:31,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:31,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:31,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:31,786 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:31,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:31,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:31,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/cc4f7501892841ca95fa2b8b6b4e94cd 2024-11-20T21:14:31,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/ed2f9165bed84c5b8b4ba260d50ffbcc as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/ed2f9165bed84c5b8b4ba260d50ffbcc 2024-11-20T21:14:31,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/ed2f9165bed84c5b8b4ba260d50ffbcc, entries=350, sequenceid=110, filesize=63.8 K 2024-11-20T21:14:31,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/238cd38c7a4f4def86e2685b8503be52 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/238cd38c7a4f4def86e2685b8503be52 2024-11-20T21:14:31,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/238cd38c7a4f4def86e2685b8503be52, entries=150, sequenceid=110, filesize=11.7 K 2024-11-20T21:14:31,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/cc4f7501892841ca95fa2b8b6b4e94cd as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/cc4f7501892841ca95fa2b8b6b4e94cd 2024-11-20T21:14:31,914 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/cc4f7501892841ca95fa2b8b6b4e94cd, entries=150, sequenceid=110, filesize=11.7 K 2024-11-20T21:14:31,916 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 917d9f6cd1adb0d5f4ed70d067a87108 in 1199ms, sequenceid=110, compaction requested=true 2024-11-20T21:14:31,916 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:31,917 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:14:31,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:14:31,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:31,918 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:14:31,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:14:31,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:31,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:14:31,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:31,923 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 166885 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:14:31,923 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 917d9f6cd1adb0d5f4ed70d067a87108/A is initiating minor compaction (all files) 2024-11-20T21:14:31,923 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 917d9f6cd1adb0d5f4ed70d067a87108/A in TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:31,923 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/f60d019128074d2a9efd4b3c22413da0, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/036c165b18f64c5fb1bd931fd43c5860, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/59951e166bff4857ab64443c3126d9dd, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/ed2f9165bed84c5b8b4ba260d50ffbcc] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp, totalSize=163.0 K 2024-11-20T21:14:31,923 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:31,923 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. files: [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/f60d019128074d2a9efd4b3c22413da0, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/036c165b18f64c5fb1bd931fd43c5860, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/59951e166bff4857ab64443c3126d9dd, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/ed2f9165bed84c5b8b4ba260d50ffbcc] 2024-11-20T21:14:31,924 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f60d019128074d2a9efd4b3c22413da0, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1732137267720 2024-11-20T21:14:31,924 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:14:31,924 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 917d9f6cd1adb0d5f4ed70d067a87108/B is initiating minor compaction (all files) 2024-11-20T21:14:31,925 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 917d9f6cd1adb0d5f4ed70d067a87108/B in TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:31,925 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/d3085ff7756847a9b4faaa620ce3725c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/5006763660844be9be657840edb975dc, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/3bb27a91313842f6808c63336c55a976, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/238cd38c7a4f4def86e2685b8503be52] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp, totalSize=47.0 K 2024-11-20T21:14:31,925 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting d3085ff7756847a9b4faaa620ce3725c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1732137267720 2024-11-20T21:14:31,925 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 036c165b18f64c5fb1bd931fd43c5860, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732137267781 2024-11-20T21:14:31,926 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 5006763660844be9be657840edb975dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732137267781 2024-11-20T21:14:31,926 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59951e166bff4857ab64443c3126d9dd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732137268448 2024-11-20T21:14:31,926 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bb27a91313842f6808c63336c55a976, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732137268448 2024-11-20T21:14:31,927 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed2f9165bed84c5b8b4ba260d50ffbcc, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732137270687 2024-11-20T21:14:31,932 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 238cd38c7a4f4def86e2685b8503be52, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732137270696 2024-11-20T21:14:31,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-20T21:14:31,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:31,947 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 917d9f6cd1adb0d5f4ed70d067a87108 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-20T21:14:31,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=A 2024-11-20T21:14:31,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:14:31,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=B 2024-11-20T21:14:31,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:14:31,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=C 2024-11-20T21:14:31,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:14:31,953 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:31,954 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 917d9f6cd1adb0d5f4ed70d067a87108#B#compaction#126 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:31,954 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/d383cb01ce774d8c96e97a32f6369621 is 50, key is test_row_0/B:col10/1732137270696/Put/seqid=0 2024-11-20T21:14:31,985 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112005c1a9b1d1fd4f8cae62042de4e50d56_917d9f6cd1adb0d5f4ed70d067a87108 store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:31,989 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112005c1a9b1d1fd4f8cae62042de4e50d56_917d9f6cd1adb0d5f4ed70d067a87108, store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:31,989 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112005c1a9b1d1fd4f8cae62042de4e50d56_917d9f6cd1adb0d5f4ed70d067a87108 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:31,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120626e32ff13d343f0ac6287cc0e27f5ef_917d9f6cd1adb0d5f4ed70d067a87108 is 50, key is test_row_0/A:col10/1732137270939/Put/seqid=0 2024-11-20T21:14:32,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741973_1149 (size=12241) 2024-11-20T21:14:32,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741974_1150 (size=4469) 2024-11-20T21:14:32,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741975_1151 (size=12304) 2024-11-20T21:14:32,021 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 917d9f6cd1adb0d5f4ed70d067a87108#A#compaction#127 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:32,022 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/4a4a90528ad34edb86eb915da6cd6de3 is 175, key is test_row_0/A:col10/1732137270696/Put/seqid=0 2024-11-20T21:14:32,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:32,029 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/d383cb01ce774d8c96e97a32f6369621 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/d383cb01ce774d8c96e97a32f6369621 2024-11-20T21:14:32,041 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120626e32ff13d343f0ac6287cc0e27f5ef_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120626e32ff13d343f0ac6287cc0e27f5ef_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:32,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/833ed27cc00047bca9055df34493ec1c, store: [table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:32,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/833ed27cc00047bca9055df34493ec1c is 175, key is test_row_0/A:col10/1732137270939/Put/seqid=0 2024-11-20T21:14:32,049 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 917d9f6cd1adb0d5f4ed70d067a87108/B of 917d9f6cd1adb0d5f4ed70d067a87108 into d383cb01ce774d8c96e97a32f6369621(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:32,049 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:32,049 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., storeName=917d9f6cd1adb0d5f4ed70d067a87108/B, priority=12, startTime=1732137271918; duration=0sec 2024-11-20T21:14:32,049 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:32,049 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:B 2024-11-20T21:14:32,049 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:14:32,062 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:14:32,062 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 917d9f6cd1adb0d5f4ed70d067a87108/C is initiating minor compaction (all files) 2024-11-20T21:14:32,062 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 917d9f6cd1adb0d5f4ed70d067a87108/C in TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:32,062 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/3e517de421c64dd8a7f06e5ab417af16, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/fce52246e3164f9d8c6b7aa16edb5861, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/54f37ed9bafb47e4be65153c408e371f, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/cc4f7501892841ca95fa2b8b6b4e94cd] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp, totalSize=47.0 K 2024-11-20T21:14:32,063 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e517de421c64dd8a7f06e5ab417af16, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=65, earliestPutTs=1732137267720 2024-11-20T21:14:32,064 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting fce52246e3164f9d8c6b7aa16edb5861, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=85, earliestPutTs=1732137267781 2024-11-20T21:14:32,065 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 54f37ed9bafb47e4be65153c408e371f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1732137268448 2024-11-20T21:14:32,066 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting cc4f7501892841ca95fa2b8b6b4e94cd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732137270696 2024-11-20T21:14:32,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741977_1153 (size=31105) 2024-11-20T21:14:32,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741976_1152 (size=31195) 2024-11-20T21:14:32,072 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=147, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/833ed27cc00047bca9055df34493ec1c 2024-11-20T21:14:32,093 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/4a4a90528ad34edb86eb915da6cd6de3 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/4a4a90528ad34edb86eb915da6cd6de3 2024-11-20T21:14:32,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/4e79197c432849a8a3053073942375ac is 50, key is test_row_0/B:col10/1732137270939/Put/seqid=0 2024-11-20T21:14:32,101 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 917d9f6cd1adb0d5f4ed70d067a87108#C#compaction#130 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:32,103 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/179fecd5651747a1acd6fddb0f6d24ac is 50, key is test_row_0/C:col10/1732137270696/Put/seqid=0 2024-11-20T21:14:32,114 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 917d9f6cd1adb0d5f4ed70d067a87108/A of 917d9f6cd1adb0d5f4ed70d067a87108 into 4a4a90528ad34edb86eb915da6cd6de3(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:32,114 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:32,114 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., storeName=917d9f6cd1adb0d5f4ed70d067a87108/A, priority=12, startTime=1732137271916; duration=0sec 2024-11-20T21:14:32,114 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:32,114 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:A 2024-11-20T21:14:32,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:32,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:32,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741978_1154 (size=12151) 2024-11-20T21:14:32,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T21:14:32,128 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/4e79197c432849a8a3053073942375ac 2024-11-20T21:14:32,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741979_1155 (size=12241) 2024-11-20T21:14:32,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:32,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137332128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:32,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137332129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:32,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:32,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137332128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:32,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,138 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:32,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:32,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137332133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:32,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:32,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:32,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137332136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:32,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:32,148 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/179fecd5651747a1acd6fddb0f6d24ac as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/179fecd5651747a1acd6fddb0f6d24ac 2024-11-20T21:14:32,163 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 917d9f6cd1adb0d5f4ed70d067a87108/C of 917d9f6cd1adb0d5f4ed70d067a87108 into 179fecd5651747a1acd6fddb0f6d24ac(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:32,163 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:32,163 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., storeName=917d9f6cd1adb0d5f4ed70d067a87108/C, priority=12, startTime=1732137271918; duration=0sec 2024-11-20T21:14:32,166 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:32,166 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:C 2024-11-20T21:14:32,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/e4cafc8b620b4e1a958b1bf8086b5776 is 50, key is test_row_0/C:col10/1732137270939/Put/seqid=0 2024-11-20T21:14:32,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741980_1156 (size=12151) 2024-11-20T21:14:32,186 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/e4cafc8b620b4e1a958b1bf8086b5776 2024-11-20T21:14:32,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/833ed27cc00047bca9055df34493ec1c as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/833ed27cc00047bca9055df34493ec1c 2024-11-20T21:14:32,205 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/833ed27cc00047bca9055df34493ec1c, entries=150, sequenceid=147, filesize=30.4 K 2024-11-20T21:14:32,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/4e79197c432849a8a3053073942375ac as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/4e79197c432849a8a3053073942375ac 2024-11-20T21:14:32,227 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/4e79197c432849a8a3053073942375ac, entries=150, sequenceid=147, filesize=11.9 K 2024-11-20T21:14:32,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/e4cafc8b620b4e1a958b1bf8086b5776 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/e4cafc8b620b4e1a958b1bf8086b5776 2024-11-20T21:14:32,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:32,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137332247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:32,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:32,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,249 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:32,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137332247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:32,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:32,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:32,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137332248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:32,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:32,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137332248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:32,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:32,253 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/e4cafc8b620b4e1a958b1bf8086b5776, entries=150, sequenceid=147, filesize=11.9 K 2024-11-20T21:14:32,255 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 917d9f6cd1adb0d5f4ed70d067a87108 in 309ms, sequenceid=147, compaction requested=false 2024-11-20T21:14:32,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:32,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:32,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-20T21:14:32,255 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-20T21:14:32,256 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:32,260 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-20T21:14:32,260 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2650 sec 2024-11-20T21:14:32,262 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.2720 sec 2024-11-20T21:14:32,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 917d9f6cd1adb0d5f4ed70d067a87108 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T21:14:32,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=A 2024-11-20T21:14:32,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:32,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=B 2024-11-20T21:14:32,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:32,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=C 2024-11-20T21:14:32,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:32,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:32,474 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120aba22873a9cc4a79b64f83a99aa8b4f5_917d9f6cd1adb0d5f4ed70d067a87108 is 50, key is test_row_0/A:col10/1732137272457/Put/seqid=0 2024-11-20T21:14:32,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741981_1157 (size=12304) 2024-11-20T21:14:32,493 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:32,498 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120aba22873a9cc4a79b64f83a99aa8b4f5_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120aba22873a9cc4a79b64f83a99aa8b4f5_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:32,500 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/e92ba6f1785d4468b3d82c31fa6e1e84, store: [table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:32,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/e92ba6f1785d4468b3d82c31fa6e1e84 is 175, key is test_row_0/A:col10/1732137272457/Put/seqid=0 2024-11-20T21:14:32,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732137272492, max timestamp=1732137272507Num uniques 859; 2024-11-20T21:14:32,511 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T21:14:32,512 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732137272492, max timestamp=1732137272509Num uniques 859; 2024-11-20T21:14:32,512 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T21:14:32,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732137272492, max timestamp=1732137272507Num uniques 859; 2024-11-20T21:14:32,514 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T21:14:32,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:32,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137332550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:32,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:32,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:32,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137332552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:32,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741982_1158 (size=31105) 2024-11-20T21:14:32,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:32,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137332555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:32,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:32,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137332555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:32,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:32,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,569 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:32,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:32,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137332667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:32,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:32,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:32,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137332678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:32,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:32,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:32,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137332681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:32,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:32,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:32,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137332681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:32,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:32,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:32,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137332887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:32,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:32,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:32,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137332887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:32,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:32,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137332898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:32,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137332898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:32,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:32,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:32,965 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=161, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/e92ba6f1785d4468b3d82c31fa6e1e84 2024-11-20T21:14:32,981 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/e8f6a45bbb4345a185c6c938c2ef46dd is 50, key is test_row_0/B:col10/1732137272457/Put/seqid=0 2024-11-20T21:14:32,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741983_1159 (size=12151) 2024-11-20T21:14:32,994 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/e8f6a45bbb4345a185c6c938c2ef46dd 2024-11-20T21:14:33,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/2b4c5d3f33574a10919f5b770554d659 is 50, key is test_row_0/C:col10/1732137272457/Put/seqid=0 2024-11-20T21:14:33,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741984_1160 (size=12151) 2024-11-20T21:14:33,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-20T21:14:33,137 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:14:33,138 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:14:33,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-20T21:14:33,141 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:14:33,141 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:14:33,142 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:14:33,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T21:14:33,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:33,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137333147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:33,149 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 0b16ab3f13b7,36987,1732137236536 for 'test_row_2' in 917d9f6cd1adb0d5f4ed70d067a87108 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T21:14:33,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:33,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:33,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:33,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:33,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137333207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:33,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:33,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137333207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:33,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:33,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:33,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:33,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137333207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:33,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:33,212 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:33,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137333211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:33,212 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:33,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T21:14:33,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-20T21:14:33,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:33,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:33,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:33,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:33,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:33,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:33,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T21:14:33,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-20T21:14:33,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:33,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:33,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:33,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:33,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:33,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:33,483 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/2b4c5d3f33574a10919f5b770554d659 2024-11-20T21:14:33,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/e92ba6f1785d4468b3d82c31fa6e1e84 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/e92ba6f1785d4468b3d82c31fa6e1e84 2024-11-20T21:14:33,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/e92ba6f1785d4468b3d82c31fa6e1e84, entries=150, sequenceid=161, filesize=30.4 K 2024-11-20T21:14:33,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/e8f6a45bbb4345a185c6c938c2ef46dd as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/e8f6a45bbb4345a185c6c938c2ef46dd 2024-11-20T21:14:33,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/e8f6a45bbb4345a185c6c938c2ef46dd, entries=150, sequenceid=161, filesize=11.9 K 2024-11-20T21:14:33,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/2b4c5d3f33574a10919f5b770554d659 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/2b4c5d3f33574a10919f5b770554d659 2024-11-20T21:14:33,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/2b4c5d3f33574a10919f5b770554d659, entries=150, sequenceid=161, filesize=11.9 K 2024-11-20T21:14:33,540 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 917d9f6cd1adb0d5f4ed70d067a87108 in 1081ms, sequenceid=161, compaction requested=true 2024-11-20T21:14:33,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:33,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:14:33,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:33,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:14:33,540 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:33,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:33,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:14:33,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T21:14:33,540 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:33,543 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:33,543 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93405 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:33,543 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 917d9f6cd1adb0d5f4ed70d067a87108/B is initiating minor compaction (all files) 2024-11-20T21:14:33,543 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 917d9f6cd1adb0d5f4ed70d067a87108/A is initiating minor compaction (all files) 2024-11-20T21:14:33,543 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 917d9f6cd1adb0d5f4ed70d067a87108/B in TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:33,543 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 917d9f6cd1adb0d5f4ed70d067a87108/A in TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:33,543 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/d383cb01ce774d8c96e97a32f6369621, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/4e79197c432849a8a3053073942375ac, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/e8f6a45bbb4345a185c6c938c2ef46dd] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp, totalSize=35.7 K 2024-11-20T21:14:33,543 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/4a4a90528ad34edb86eb915da6cd6de3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/833ed27cc00047bca9055df34493ec1c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/e92ba6f1785d4468b3d82c31fa6e1e84] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp, totalSize=91.2 K 2024-11-20T21:14:33,543 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:33,543 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. files: [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/4a4a90528ad34edb86eb915da6cd6de3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/833ed27cc00047bca9055df34493ec1c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/e92ba6f1785d4468b3d82c31fa6e1e84] 2024-11-20T21:14:33,544 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting d383cb01ce774d8c96e97a32f6369621, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732137270696 2024-11-20T21:14:33,544 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a4a90528ad34edb86eb915da6cd6de3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732137270696 2024-11-20T21:14:33,544 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e79197c432849a8a3053073942375ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732137270925 2024-11-20T21:14:33,544 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 833ed27cc00047bca9055df34493ec1c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732137270925 2024-11-20T21:14:33,545 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8f6a45bbb4345a185c6c938c2ef46dd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732137272130 2024-11-20T21:14:33,545 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting e92ba6f1785d4468b3d82c31fa6e1e84, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732137272130 2024-11-20T21:14:33,584 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 917d9f6cd1adb0d5f4ed70d067a87108#B#compaction#135 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:33,584 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/8a8fbed3603044a4839147270e16621c is 50, key is test_row_0/B:col10/1732137272457/Put/seqid=0 2024-11-20T21:14:33,594 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:33,624 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411201e0e233c76b840a6849d63dfe88932d7_917d9f6cd1adb0d5f4ed70d067a87108 store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:33,626 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411201e0e233c76b840a6849d63dfe88932d7_917d9f6cd1adb0d5f4ed70d067a87108, store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:33,626 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201e0e233c76b840a6849d63dfe88932d7_917d9f6cd1adb0d5f4ed70d067a87108 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:33,632 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-20T21:14:33,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:33,633 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 917d9f6cd1adb0d5f4ed70d067a87108 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-20T21:14:33,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=A 2024-11-20T21:14:33,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:14:33,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=B 2024-11-20T21:14:33,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:14:33,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=C 2024-11-20T21:14:33,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:14:33,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741985_1161 (size=12493) 2024-11-20T21:14:33,679 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/8a8fbed3603044a4839147270e16621c as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/8a8fbed3603044a4839147270e16621c 2024-11-20T21:14:33,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741986_1162 (size=4469) 2024-11-20T21:14:33,687 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 917d9f6cd1adb0d5f4ed70d067a87108#A#compaction#136 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:33,688 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/159e3e7b47a44f5f8d9c94f8b653eb9f is 175, key is test_row_0/A:col10/1732137272457/Put/seqid=0 2024-11-20T21:14:33,692 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 917d9f6cd1adb0d5f4ed70d067a87108/B of 917d9f6cd1adb0d5f4ed70d067a87108 into 8a8fbed3603044a4839147270e16621c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:33,692 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:33,692 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., storeName=917d9f6cd1adb0d5f4ed70d067a87108/B, priority=13, startTime=1732137273540; duration=0sec 2024-11-20T21:14:33,692 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:33,692 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:B 2024-11-20T21:14:33,692 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:33,694 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:33,694 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 917d9f6cd1adb0d5f4ed70d067a87108/C is initiating minor compaction (all files) 2024-11-20T21:14:33,694 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 917d9f6cd1adb0d5f4ed70d067a87108/C in TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:33,694 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/179fecd5651747a1acd6fddb0f6d24ac, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/e4cafc8b620b4e1a958b1bf8086b5776, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/2b4c5d3f33574a10919f5b770554d659] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp, totalSize=35.7 K 2024-11-20T21:14:33,695 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 179fecd5651747a1acd6fddb0f6d24ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732137270696 2024-11-20T21:14:33,696 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4cafc8b620b4e1a958b1bf8086b5776, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732137270925 2024-11-20T21:14:33,696 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b4c5d3f33574a10919f5b770554d659, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732137272130 2024-11-20T21:14:33,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411204a7b1c0394e44f92b6a58efc163b7571_917d9f6cd1adb0d5f4ed70d067a87108 is 50, key is test_row_0/A:col10/1732137272552/Put/seqid=0 2024-11-20T21:14:33,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:33,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:33,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741987_1163 (size=31447) 2024-11-20T21:14:33,731 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 917d9f6cd1adb0d5f4ed70d067a87108#C#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:33,732 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/ac35c7516a654afcbb0825cc745e2bfb is 50, key is test_row_0/C:col10/1732137272457/Put/seqid=0 2024-11-20T21:14:33,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741988_1164 (size=12304) 2024-11-20T21:14:33,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741989_1165 (size=12493) 2024-11-20T21:14:33,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:33,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137333736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:33,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:33,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:33,744 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:33,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:33,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137333738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:33,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:33,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:33,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137333738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:33,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:33,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:33,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:33,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137333740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:33,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:33,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:33,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:33,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:33,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:33,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:33,752 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/ac35c7516a654afcbb0825cc745e2bfb as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/ac35c7516a654afcbb0825cc745e2bfb 2024-11-20T21:14:33,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T21:14:33,772 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 917d9f6cd1adb0d5f4ed70d067a87108/C of 917d9f6cd1adb0d5f4ed70d067a87108 into ac35c7516a654afcbb0825cc745e2bfb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:33,772 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:33,772 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., storeName=917d9f6cd1adb0d5f4ed70d067a87108/C, priority=13, startTime=1732137273540; duration=0sec 2024-11-20T21:14:33,773 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:33,773 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:C 2024-11-20T21:14:33,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:33,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137333847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:33,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:33,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:33,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:33,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:33,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137333857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:33,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:33,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:33,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:33,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137333857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:33,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:33,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:33,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137333857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:33,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:33,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:33,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:33,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:33,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:33,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:34,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:34,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:34,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137334069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:34,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137334066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:34,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:34,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137334069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:34,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:34,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:34,074 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:34,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:34,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137334071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:34,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:34,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:34,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:34,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:34,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:34,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:34,076 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:34,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:34,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,157 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/159e3e7b47a44f5f8d9c94f8b653eb9f as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/159e3e7b47a44f5f8d9c94f8b653eb9f 2024-11-20T21:14:34,179 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 917d9f6cd1adb0d5f4ed70d067a87108/A of 917d9f6cd1adb0d5f4ed70d067a87108 into 159e3e7b47a44f5f8d9c94f8b653eb9f(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:34,179 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:34,179 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., storeName=917d9f6cd1adb0d5f4ed70d067a87108/A, priority=13, startTime=1732137273540; duration=0sec 2024-11-20T21:14:34,180 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:34,180 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:A 2024-11-20T21:14:34,189 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411204a7b1c0394e44f92b6a58efc163b7571_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411204a7b1c0394e44f92b6a58efc163b7571_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:34,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/cb9950d0c6a946a48f13d8d6fda639c7, store: [table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:34,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/cb9950d0c6a946a48f13d8d6fda639c7 is 175, key is test_row_0/A:col10/1732137272552/Put/seqid=0 2024-11-20T21:14:34,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741990_1166 (size=31105) 2024-11-20T21:14:34,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T21:14:34,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:34,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137334387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:34,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:34,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:34,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:34,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:34,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137334388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:34,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:34,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:34,396 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:34,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:34,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137334389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:34,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:34,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:34,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:34,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:34,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137334389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:34,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:34,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:34,398 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:34,629 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/cb9950d0c6a946a48f13d8d6fda639c7 2024-11-20T21:14:34,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/5fc403c4aa3c4a3590e21b84e18b8f92 is 50, key is test_row_0/B:col10/1732137272552/Put/seqid=0 2024-11-20T21:14:34,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741991_1167 (size=12151) 2024-11-20T21:14:34,643 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/5fc403c4aa3c4a3590e21b84e18b8f92 2024-11-20T21:14:34,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/393a26f23d1345978e4a89e16780bb46 is 50, key is test_row_0/C:col10/1732137272552/Put/seqid=0 2024-11-20T21:14:34,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741992_1168 (size=12151) 2024-11-20T21:14:34,675 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/393a26f23d1345978e4a89e16780bb46 2024-11-20T21:14:34,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/cb9950d0c6a946a48f13d8d6fda639c7 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/cb9950d0c6a946a48f13d8d6fda639c7 2024-11-20T21:14:34,701 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/cb9950d0c6a946a48f13d8d6fda639c7, entries=150, sequenceid=196, filesize=30.4 K 2024-11-20T21:14:34,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/5fc403c4aa3c4a3590e21b84e18b8f92 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/5fc403c4aa3c4a3590e21b84e18b8f92 2024-11-20T21:14:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,709 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/5fc403c4aa3c4a3590e21b84e18b8f92, entries=150, sequenceid=196, filesize=11.9 K 2024-11-20T21:14:34,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/393a26f23d1345978e4a89e16780bb46 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/393a26f23d1345978e4a89e16780bb46 2024-11-20T21:14:34,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,716 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/393a26f23d1345978e4a89e16780bb46, entries=150, sequenceid=196, filesize=11.9 K 2024-11-20T21:14:34,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,717 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 917d9f6cd1adb0d5f4ed70d067a87108 in 1084ms, sequenceid=196, compaction requested=false 2024-11-20T21:14:34,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:34,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:34,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-20T21:14:34,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-20T21:14:34,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,720 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-20T21:14:34,720 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5770 sec 2024-11-20T21:14:34,722 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.5830 sec 2024-11-20T21:14:34,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 917d9f6cd1adb0d5f4ed70d067a87108 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T21:14:34,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=A 2024-11-20T21:14:34,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:34,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=B 2024-11-20T21:14:34,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:34,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=C 2024-11-20T21:14:34,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:34,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208db8dbebad19491f9d6688ada8e64248_917d9f6cd1adb0d5f4ed70d067a87108 is 50, key is test_row_0/A:col10/1732137274911/Put/seqid=0 2024-11-20T21:14:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741993_1169 (size=12304) 2024-11-20T21:14:34,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,948 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:34,954 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208db8dbebad19491f9d6688ada8e64248_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208db8dbebad19491f9d6688ada8e64248_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:34,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:34,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137334947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:34,956 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/a40ffb66b0004ec28f4185f8ac836f84, store: [table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:34,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/a40ffb66b0004ec28f4185f8ac836f84 is 175, key is test_row_0/A:col10/1732137274911/Put/seqid=0 2024-11-20T21:14:34,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:34,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:34,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137334951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:34,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:34,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:34,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137334955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:34,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:34,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:34,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137334953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:34,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:34,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:34,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:34,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:34,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:34,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:34,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741994_1170 (size=31101) 2024-11-20T21:14:34,993 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=211, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/a40ffb66b0004ec28f4185f8ac836f84 2024-11-20T21:14:35,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/28ce742cee1d4b85be7a0b33cef03a82 is 50, key is test_row_0/B:col10/1732137274911/Put/seqid=0 2024-11-20T21:14:35,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741995_1171 (size=9757) 2024-11-20T21:14:35,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/28ce742cee1d4b85be7a0b33cef03a82 2024-11-20T21:14:35,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/31f0bf8b7fb248b698ca6278d59eb921 is 50, key is test_row_0/C:col10/1732137274911/Put/seqid=0 2024-11-20T21:14:35,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:35,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137335057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:35,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:35,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:35,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137335066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:35,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,068 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,069 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:35,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:35,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137335066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:35,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:35,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,072 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:35,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137335066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:35,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,072 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:35,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741996_1172 (size=9757) 2024-11-20T21:14:35,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:35,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137335166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:35,169 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 0b16ab3f13b7,36987,1732137236536 for 'test_row_2' in 917d9f6cd1adb0d5f4ed70d067a87108 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T21:14:35,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:35,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:35,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137335267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:35,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:35,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137335277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:35,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:35,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:35,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137335278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:35,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:35,285 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:35,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137335278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:35,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,286 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:35,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-20T21:14:35,293 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:14:35,294 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:14:35,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-20T21:14:35,299 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:14:35,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T21:14:35,300 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:14:35,301 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:14:35,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T21:14:35,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T21:14:35,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:35,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:35,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:35,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:35,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:35,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:35,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/31f0bf8b7fb248b698ca6278d59eb921 2024-11-20T21:14:35,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/a40ffb66b0004ec28f4185f8ac836f84 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/a40ffb66b0004ec28f4185f8ac836f84 2024-11-20T21:14:35,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/a40ffb66b0004ec28f4185f8ac836f84, entries=150, sequenceid=211, filesize=30.4 K 2024-11-20T21:14:35,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/28ce742cee1d4b85be7a0b33cef03a82 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/28ce742cee1d4b85be7a0b33cef03a82 2024-11-20T21:14:35,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/28ce742cee1d4b85be7a0b33cef03a82, entries=100, sequenceid=211, filesize=9.5 K 2024-11-20T21:14:35,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/31f0bf8b7fb248b698ca6278d59eb921 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/31f0bf8b7fb248b698ca6278d59eb921 2024-11-20T21:14:35,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/31f0bf8b7fb248b698ca6278d59eb921, entries=100, sequenceid=211, filesize=9.5 K 2024-11-20T21:14:35,553 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 917d9f6cd1adb0d5f4ed70d067a87108 in 641ms, sequenceid=211, compaction requested=true 2024-11-20T21:14:35,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:35,553 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:35,554 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:35,554 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 917d9f6cd1adb0d5f4ed70d067a87108/A is initiating minor compaction (all files) 2024-11-20T21:14:35,554 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 917d9f6cd1adb0d5f4ed70d067a87108/A in TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:35,554 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/159e3e7b47a44f5f8d9c94f8b653eb9f, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/cb9950d0c6a946a48f13d8d6fda639c7, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/a40ffb66b0004ec28f4185f8ac836f84] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp, totalSize=91.5 K 2024-11-20T21:14:35,554 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:35,554 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. files: [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/159e3e7b47a44f5f8d9c94f8b653eb9f, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/cb9950d0c6a946a48f13d8d6fda639c7, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/a40ffb66b0004ec28f4185f8ac836f84] 2024-11-20T21:14:35,555 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 159e3e7b47a44f5f8d9c94f8b653eb9f, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732137272130 2024-11-20T21:14:35,555 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb9950d0c6a946a48f13d8d6fda639c7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732137272534 2024-11-20T21:14:35,556 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting a40ffb66b0004ec28f4185f8ac836f84, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732137273736 2024-11-20T21:14:35,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:14:35,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:35,564 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:35,566 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:35,566 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 917d9f6cd1adb0d5f4ed70d067a87108/B is initiating minor compaction (all files) 2024-11-20T21:14:35,566 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 917d9f6cd1adb0d5f4ed70d067a87108/B in TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:35,566 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/8a8fbed3603044a4839147270e16621c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/5fc403c4aa3c4a3590e21b84e18b8f92, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/28ce742cee1d4b85be7a0b33cef03a82] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp, totalSize=33.6 K 2024-11-20T21:14:35,567 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a8fbed3603044a4839147270e16621c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732137272130 2024-11-20T21:14:35,567 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fc403c4aa3c4a3590e21b84e18b8f92, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732137272534 2024-11-20T21:14:35,568 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 28ce742cee1d4b85be7a0b33cef03a82, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732137273736 2024-11-20T21:14:35,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:14:35,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:35,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:14:35,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:35,581 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:35,591 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 917d9f6cd1adb0d5f4ed70d067a87108#B#compaction#145 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:35,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732137274948, max timestamp=1732137275588Num uniques 859; 2024-11-20T21:14:35,592 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T21:14:35,592 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/174056d93c1c46d08393d828357eb0c1 is 50, key is test_row_0/B:col10/1732137274911/Put/seqid=0 2024-11-20T21:14:35,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732137274948, max timestamp=1732137275588Num uniques 859; 2024-11-20T21:14:35,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732137274948, max timestamp=1732137275588Num uniques 859; 2024-11-20T21:14:35,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T21:14:35,595 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T21:14:35,605 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120113469c0878041b4be8aa5b88793d187_917d9f6cd1adb0d5f4ed70d067a87108 store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:35,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:35,607 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120113469c0878041b4be8aa5b88793d187_917d9f6cd1adb0d5f4ed70d067a87108, store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:35,608 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120113469c0878041b4be8aa5b88793d187_917d9f6cd1adb0d5f4ed70d067a87108 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:35,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 917d9f6cd1adb0d5f4ed70d067a87108 3/3 column families, dataSize=60.38 KB heapSize=165.84 KB 2024-11-20T21:14:35,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T21:14:35,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=A 2024-11-20T21:14:35,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:14:35,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=B 2024-11-20T21:14:35,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:14:35,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=C 2024-11-20T21:14:35,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:35,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:14:35,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:35,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:35,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:35,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:35,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:35,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T21:14:35,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741997_1173 (size=12595) 2024-11-20T21:14:35,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120cb961bdddf01402b87da7b72642ae17b_917d9f6cd1adb0d5f4ed70d067a87108 is 50, key is test_row_0/A:col10/1732137275604/Put/seqid=0 2024-11-20T21:14:35,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741998_1174 (size=4469) 2024-11-20T21:14:35,664 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/174056d93c1c46d08393d828357eb0c1 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/174056d93c1c46d08393d828357eb0c1 2024-11-20T21:14:35,671 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 917d9f6cd1adb0d5f4ed70d067a87108/B of 917d9f6cd1adb0d5f4ed70d067a87108 into 174056d93c1c46d08393d828357eb0c1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:35,671 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:35,671 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., storeName=917d9f6cd1adb0d5f4ed70d067a87108/B, priority=13, startTime=1732137275564; duration=0sec 2024-11-20T21:14:35,671 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:35,672 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:B 2024-11-20T21:14:35,672 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:35,673 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:35,673 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 917d9f6cd1adb0d5f4ed70d067a87108/C is initiating minor compaction (all files) 2024-11-20T21:14:35,673 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 917d9f6cd1adb0d5f4ed70d067a87108/C in TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:35,673 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/ac35c7516a654afcbb0825cc745e2bfb, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/393a26f23d1345978e4a89e16780bb46, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/31f0bf8b7fb248b698ca6278d59eb921] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp, totalSize=33.6 K 2024-11-20T21:14:35,674 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting ac35c7516a654afcbb0825cc745e2bfb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1732137272130 2024-11-20T21:14:35,675 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 393a26f23d1345978e4a89e16780bb46, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1732137272534 2024-11-20T21:14:35,675 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 31f0bf8b7fb248b698ca6278d59eb921, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732137273736 2024-11-20T21:14:35,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:35,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137335666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:35,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:35,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:35,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137335668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:35,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:35,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741999_1175 (size=19774) 2024-11-20T21:14:35,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:35,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137335680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:35,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,695 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:35,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:35,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137335682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:35,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:35,704 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 917d9f6cd1adb0d5f4ed70d067a87108#C#compaction#147 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:35,705 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/912795a83ec64752a89bdcc80c4b1518 is 50, key is test_row_0/C:col10/1732137274911/Put/seqid=0 2024-11-20T21:14:35,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742000_1176 (size=12595) 2024-11-20T21:14:35,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T21:14:35,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:35,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:35,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:35,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:35,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:35,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:35,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:35,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137335787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:35,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,792 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:35,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137335798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:35,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,806 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:35,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:35,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137335808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:35,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:35,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:35,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137335808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:35,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:35,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:35,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T21:14:35,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:35,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:35,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:35,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:35,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T21:14:36,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:36,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137335997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:36,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:36,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:36,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:36,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:36,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137336016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:36,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:36,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:36,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:36,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:36,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137336027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:36,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:36,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:36,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:36,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:36,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137336027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:36,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:36,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:36,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:36,062 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 917d9f6cd1adb0d5f4ed70d067a87108#A#compaction#144 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:36,063 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/10d4be816bda4cf081e9c90c40228296 is 175, key is test_row_0/A:col10/1732137274911/Put/seqid=0 2024-11-20T21:14:36,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742001_1177 (size=31656) 2024-11-20T21:14:36,091 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T21:14:36,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:36,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:36,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:36,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:36,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:36,092 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:36,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:36,096 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/10d4be816bda4cf081e9c90c40228296 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/10d4be816bda4cf081e9c90c40228296 2024-11-20T21:14:36,100 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120cb961bdddf01402b87da7b72642ae17b_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120cb961bdddf01402b87da7b72642ae17b_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:36,101 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/3e2030e1ed2b4b9da565c417ea3458f3, store: [table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:36,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/3e2030e1ed2b4b9da565c417ea3458f3 is 175, key is test_row_0/A:col10/1732137275604/Put/seqid=0 2024-11-20T21:14:36,104 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 917d9f6cd1adb0d5f4ed70d067a87108/A of 917d9f6cd1adb0d5f4ed70d067a87108 into 10d4be816bda4cf081e9c90c40228296(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:36,104 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:36,104 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., storeName=917d9f6cd1adb0d5f4ed70d067a87108/A, priority=13, startTime=1732137275553; duration=0sec 2024-11-20T21:14:36,104 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:36,105 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:A 2024-11-20T21:14:36,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742002_1178 (size=57033) 2024-11-20T21:14:36,154 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/912795a83ec64752a89bdcc80c4b1518 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/912795a83ec64752a89bdcc80c4b1518 2024-11-20T21:14:36,160 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 917d9f6cd1adb0d5f4ed70d067a87108/C of 917d9f6cd1adb0d5f4ed70d067a87108 into 912795a83ec64752a89bdcc80c4b1518(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:36,160 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:36,160 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., storeName=917d9f6cd1adb0d5f4ed70d067a87108/C, priority=13, startTime=1732137275577; duration=0sec 2024-11-20T21:14:36,160 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:36,161 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:C 2024-11-20T21:14:36,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T21:14:36,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:36,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:36,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:36,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:36,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:36,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:36,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:36,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137336307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:36,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:36,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:36,311 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:36,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:36,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137336327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:36,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:36,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:36,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137336337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:36,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:36,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137336337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:36,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:36,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T21:14:36,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:36,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:36,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:36,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:36,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:36,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:36,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T21:14:36,513 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/3e2030e1ed2b4b9da565c417ea3458f3 2024-11-20T21:14:36,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/4bdd7a11231d4931919f6861fc9e2df5 is 50, key is test_row_0/B:col10/1732137275604/Put/seqid=0 2024-11-20T21:14:36,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T21:14:36,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:36,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:36,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:36,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:36,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:36,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742003_1179 (size=12151) 2024-11-20T21:14:36,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/4bdd7a11231d4931919f6861fc9e2df5 2024-11-20T21:14:36,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/8a2e960de3464409b1430bb4b5735428 is 50, key is test_row_0/C:col10/1732137275604/Put/seqid=0 2024-11-20T21:14:36,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742004_1180 (size=12151) 2024-11-20T21:14:36,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/8a2e960de3464409b1430bb4b5735428 2024-11-20T21:14:36,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/3e2030e1ed2b4b9da565c417ea3458f3 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/3e2030e1ed2b4b9da565c417ea3458f3 2024-11-20T21:14:36,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/3e2030e1ed2b4b9da565c417ea3458f3, entries=300, sequenceid=238, filesize=55.7 K 2024-11-20T21:14:36,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/4bdd7a11231d4931919f6861fc9e2df5 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/4bdd7a11231d4931919f6861fc9e2df5 2024-11-20T21:14:36,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/4bdd7a11231d4931919f6861fc9e2df5, entries=150, sequenceid=238, filesize=11.9 K 2024-11-20T21:14:36,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/8a2e960de3464409b1430bb4b5735428 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/8a2e960de3464409b1430bb4b5735428 2024-11-20T21:14:36,673 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/8a2e960de3464409b1430bb4b5735428, entries=150, sequenceid=238, filesize=11.9 K 2024-11-20T21:14:36,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~185.72 KB/190176, currentSize=114.05 KB/116790 for 917d9f6cd1adb0d5f4ed70d067a87108 in 1067ms, sequenceid=238, compaction requested=false 2024-11-20T21:14:36,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:36,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-20T21:14:36,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:36,720 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 917d9f6cd1adb0d5f4ed70d067a87108 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T21:14:36,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=A 2024-11-20T21:14:36,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:36,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=B 2024-11-20T21:14:36,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:36,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=C 2024-11-20T21:14:36,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:36,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120cbba340b1c424f7286f228178faa4218_917d9f6cd1adb0d5f4ed70d067a87108 is 50, key is test_row_0/A:col10/1732137275661/Put/seqid=0 2024-11-20T21:14:36,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742005_1181 (size=12354) 2024-11-20T21:14:36,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:36,771 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120cbba340b1c424f7286f228178faa4218_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120cbba340b1c424f7286f228178faa4218_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:36,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/8f5303fa4c964d47ad1a99fcedd341b9, store: [table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:36,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/8f5303fa4c964d47ad1a99fcedd341b9 is 175, key is test_row_0/A:col10/1732137275661/Put/seqid=0 2024-11-20T21:14:36,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742006_1182 (size=31155) 2024-11-20T21:14:36,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:36,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:36,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:36,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137336848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:36,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:36,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:36,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:36,855 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:36,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137336848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:36,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:36,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:36,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:36,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:36,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137336848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:36,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:36,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:36,855 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:36,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:36,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137336853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:36,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:36,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:36,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:36,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:36,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137336957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:36,967 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:36,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:36,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:36,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:36,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137336976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:37,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:37,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:37,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:37,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:37,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137337177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:37,178 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:37,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:37,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:37,191 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=261, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/8f5303fa4c964d47ad1a99fcedd341b9 2024-11-20T21:14:37,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/8dc85ee02f954cd5af0d695785e22534 is 50, key is test_row_0/B:col10/1732137275661/Put/seqid=0 2024-11-20T21:14:37,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:37,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137337218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:37,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:37,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:37,225 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:37,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742007_1183 (size=12201) 2024-11-20T21:14:37,232 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/8dc85ee02f954cd5af0d695785e22534 2024-11-20T21:14:37,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/882d6fe22a944cfea0118c8f0bad6398 is 50, key is test_row_0/C:col10/1732137275661/Put/seqid=0 2024-11-20T21:14:37,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742008_1184 (size=12201) 2024-11-20T21:14:37,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T21:14:37,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:37,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137337489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:37,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:37,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:37,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:37,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:37,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137337541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:37,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:37,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:37,543 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:37,673 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/882d6fe22a944cfea0118c8f0bad6398 2024-11-20T21:14:37,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/8f5303fa4c964d47ad1a99fcedd341b9 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/8f5303fa4c964d47ad1a99fcedd341b9 2024-11-20T21:14:37,695 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/8f5303fa4c964d47ad1a99fcedd341b9, entries=150, sequenceid=261, filesize=30.4 K 2024-11-20T21:14:37,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/8dc85ee02f954cd5af0d695785e22534 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/8dc85ee02f954cd5af0d695785e22534 2024-11-20T21:14:37,702 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/8dc85ee02f954cd5af0d695785e22534, entries=150, sequenceid=261, filesize=11.9 K 2024-11-20T21:14:37,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/882d6fe22a944cfea0118c8f0bad6398 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/882d6fe22a944cfea0118c8f0bad6398 2024-11-20T21:14:37,709 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/882d6fe22a944cfea0118c8f0bad6398, entries=150, sequenceid=261, filesize=11.9 K 2024-11-20T21:14:37,710 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 917d9f6cd1adb0d5f4ed70d067a87108 in 990ms, sequenceid=261, compaction requested=true 2024-11-20T21:14:37,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:37,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:37,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-20T21:14:37,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-20T21:14:37,715 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-20T21:14:37,715 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4110 sec 2024-11-20T21:14:37,719 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.4240 sec 2024-11-20T21:14:37,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:37,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 917d9f6cd1adb0d5f4ed70d067a87108 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-20T21:14:37,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=A 2024-11-20T21:14:37,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:37,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=B 2024-11-20T21:14:37,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:37,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=C 2024-11-20T21:14:37,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:37,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ca6a534f13d943b7838e7d0754804a69_917d9f6cd1adb0d5f4ed70d067a87108 is 50, key is test_row_0/A:col10/1732137276841/Put/seqid=0 2024-11-20T21:14:37,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742009_1185 (size=14994) 2024-11-20T21:14:37,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:37,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137337920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:37,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:37,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:37,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137337928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:37,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:37,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:37,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:37,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:37,929 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:38,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137338007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:38,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:38,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:38,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:38,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:38,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137338036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:38,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:38,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:38,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:38,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:38,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137338036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:38,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:38,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:38,039 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:38,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:38,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137338057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:38,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:38,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:38,059 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:38,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:38,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137338247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:38,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:38,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137338247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:38,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:38,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:38,253 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:38,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:38,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:38,253 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:38,336 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:38,361 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ca6a534f13d943b7838e7d0754804a69_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ca6a534f13d943b7838e7d0754804a69_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:38,364 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/c3a2027d114c4d179d794c21fab5c5fc, store: [table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:38,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/c3a2027d114c4d179d794c21fab5c5fc is 175, key is test_row_0/A:col10/1732137276841/Put/seqid=0 2024-11-20T21:14:38,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742010_1186 (size=39949) 2024-11-20T21:14:38,386 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/c3a2027d114c4d179d794c21fab5c5fc 2024-11-20T21:14:38,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/875c2ab7b2a9487188e31614c0b22751 is 50, key is test_row_0/B:col10/1732137276841/Put/seqid=0 2024-11-20T21:14:38,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742011_1187 (size=12301) 2024-11-20T21:14:38,414 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/875c2ab7b2a9487188e31614c0b22751 2024-11-20T21:14:38,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/85d8701f7bf24b9ab6c4f8de75cef0d2 is 50, key is test_row_0/C:col10/1732137276841/Put/seqid=0 2024-11-20T21:14:38,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742012_1188 (size=12301) 2024-11-20T21:14:38,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/85d8701f7bf24b9ab6c4f8de75cef0d2 2024-11-20T21:14:38,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/c3a2027d114c4d179d794c21fab5c5fc as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/c3a2027d114c4d179d794c21fab5c5fc 2024-11-20T21:14:38,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/c3a2027d114c4d179d794c21fab5c5fc, entries=200, sequenceid=274, filesize=39.0 K 2024-11-20T21:14:38,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/875c2ab7b2a9487188e31614c0b22751 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/875c2ab7b2a9487188e31614c0b22751 2024-11-20T21:14:38,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/875c2ab7b2a9487188e31614c0b22751, entries=150, sequenceid=274, filesize=12.0 K 2024-11-20T21:14:38,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/85d8701f7bf24b9ab6c4f8de75cef0d2 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/85d8701f7bf24b9ab6c4f8de75cef0d2 2024-11-20T21:14:38,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/85d8701f7bf24b9ab6c4f8de75cef0d2, entries=150, sequenceid=274, filesize=12.0 K 2024-11-20T21:14:38,523 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 917d9f6cd1adb0d5f4ed70d067a87108 in 652ms, sequenceid=274, compaction requested=true 2024-11-20T21:14:38,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:38,524 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:14:38,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:14:38,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:38,524 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:14:38,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:14:38,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:38,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:14:38,524 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:38,525 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 159793 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:14:38,525 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 917d9f6cd1adb0d5f4ed70d067a87108/A is initiating minor compaction (all files) 2024-11-20T21:14:38,525 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 917d9f6cd1adb0d5f4ed70d067a87108/A in TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:38,525 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/10d4be816bda4cf081e9c90c40228296, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/3e2030e1ed2b4b9da565c417ea3458f3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/8f5303fa4c964d47ad1a99fcedd341b9, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/c3a2027d114c4d179d794c21fab5c5fc] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp, totalSize=156.0 K 2024-11-20T21:14:38,526 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49248 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:14:38,526 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:38,526 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 917d9f6cd1adb0d5f4ed70d067a87108/B is initiating minor compaction (all files) 2024-11-20T21:14:38,526 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. files: [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/10d4be816bda4cf081e9c90c40228296, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/3e2030e1ed2b4b9da565c417ea3458f3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/8f5303fa4c964d47ad1a99fcedd341b9, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/c3a2027d114c4d179d794c21fab5c5fc] 2024-11-20T21:14:38,526 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 917d9f6cd1adb0d5f4ed70d067a87108/B in TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:38,526 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/174056d93c1c46d08393d828357eb0c1, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/4bdd7a11231d4931919f6861fc9e2df5, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/8dc85ee02f954cd5af0d695785e22534, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/875c2ab7b2a9487188e31614c0b22751] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp, totalSize=48.1 K 2024-11-20T21:14:38,526 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10d4be816bda4cf081e9c90c40228296, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732137272534 2024-11-20T21:14:38,526 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 174056d93c1c46d08393d828357eb0c1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732137272534 2024-11-20T21:14:38,527 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e2030e1ed2b4b9da565c417ea3458f3, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732137275588 2024-11-20T21:14:38,527 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bdd7a11231d4931919f6861fc9e2df5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732137275588 2024-11-20T21:14:38,527 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f5303fa4c964d47ad1a99fcedd341b9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1732137275658 2024-11-20T21:14:38,527 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dc85ee02f954cd5af0d695785e22534, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1732137275658 2024-11-20T21:14:38,528 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3a2027d114c4d179d794c21fab5c5fc, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732137276838 2024-11-20T21:14:38,528 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 875c2ab7b2a9487188e31614c0b22751, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732137276841 2024-11-20T21:14:38,541 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:38,542 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 917d9f6cd1adb0d5f4ed70d067a87108#B#compaction#156 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:38,543 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/0fade51b0a324076bac7484012c068d5 is 50, key is test_row_0/B:col10/1732137276841/Put/seqid=0 2024-11-20T21:14:38,549 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120ce363a1a70b84fbaaf482d3a6c5e9893_917d9f6cd1adb0d5f4ed70d067a87108 store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:38,551 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120ce363a1a70b84fbaaf482d3a6c5e9893_917d9f6cd1adb0d5f4ed70d067a87108, store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:38,551 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ce363a1a70b84fbaaf482d3a6c5e9893_917d9f6cd1adb0d5f4ed70d067a87108 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:38,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 917d9f6cd1adb0d5f4ed70d067a87108 3/3 column families, dataSize=114.05 KB heapSize=339.66 KB 2024-11-20T21:14:38,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:38,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732137277908, max timestamp=1732137278560Num uniques 859; 2024-11-20T21:14:38,566 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T21:14:38,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=A 2024-11-20T21:14:38,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:14:38,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=B 2024-11-20T21:14:38,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:14:38,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=C 2024-11-20T21:14:38,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:14:38,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742014_1190 (size=4469) 2024-11-20T21:14:38,585 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 917d9f6cd1adb0d5f4ed70d067a87108#A#compaction#157 average throughput is 0.56 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:38,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742013_1189 (size=12881) 2024-11-20T21:14:38,586 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/b644816e0e084f4384c233342c1ac260 is 175, key is test_row_0/A:col10/1732137276841/Put/seqid=0 2024-11-20T21:14:38,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202db99cc3d72d4e6fb29ab9dab163fd60_917d9f6cd1adb0d5f4ed70d067a87108 is 50, key is test_row_0/A:col10/1732137278560/Put/seqid=0 2024-11-20T21:14:38,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742015_1191 (size=31835) 2024-11-20T21:14:38,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742016_1192 (size=17534) 2024-11-20T21:14:38,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137338628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:38,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:38,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:38,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:38,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:38,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137338629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:38,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:38,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:38,634 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:38,635 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:38,642 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202db99cc3d72d4e6fb29ab9dab163fd60_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202db99cc3d72d4e6fb29ab9dab163fd60_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:38,644 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/16007388d96047288b7787e6ccc39c7e, store: [table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:38,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/16007388d96047288b7787e6ccc39c7e is 175, key is test_row_0/A:col10/1732137278560/Put/seqid=0 2024-11-20T21:14:38,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742017_1193 (size=48639) 2024-11-20T21:14:38,669 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=295, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/16007388d96047288b7787e6ccc39c7e 2024-11-20T21:14:38,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/1c8c41e0fd4446b999bb1b84baa94621 is 50, key is test_row_0/B:col10/1732137278560/Put/seqid=0 2024-11-20T21:14:38,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742018_1194 (size=12301) 2024-11-20T21:14:38,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/1c8c41e0fd4446b999bb1b84baa94621 2024-11-20T21:14:38,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/1e42c9dc636248398ef07fecf42a9348 is 50, key is test_row_0/C:col10/1732137278560/Put/seqid=0 2024-11-20T21:14:38,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742019_1195 (size=12301) 2024-11-20T21:14:38,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137338737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:38,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:38,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137338737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:38,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:38,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:38,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137338948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:38,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:38,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137338948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:38,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:38,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:38,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:38,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:38,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:38,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:38,991 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/0fade51b0a324076bac7484012c068d5 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/0fade51b0a324076bac7484012c068d5 2024-11-20T21:14:38,997 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 917d9f6cd1adb0d5f4ed70d067a87108/B of 917d9f6cd1adb0d5f4ed70d067a87108 into 0fade51b0a324076bac7484012c068d5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:38,997 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:38,997 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., storeName=917d9f6cd1adb0d5f4ed70d067a87108/B, priority=12, startTime=1732137278524; duration=0sec 2024-11-20T21:14:38,997 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:38,998 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:B 2024-11-20T21:14:38,998 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:14:38,999 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49248 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:14:38,999 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 917d9f6cd1adb0d5f4ed70d067a87108/C is initiating minor compaction (all files) 2024-11-20T21:14:38,999 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 917d9f6cd1adb0d5f4ed70d067a87108/C in TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:38,999 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/912795a83ec64752a89bdcc80c4b1518, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/8a2e960de3464409b1430bb4b5735428, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/882d6fe22a944cfea0118c8f0bad6398, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/85d8701f7bf24b9ab6c4f8de75cef0d2] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp, totalSize=48.1 K 2024-11-20T21:14:39,000 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 912795a83ec64752a89bdcc80c4b1518, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732137272534 2024-11-20T21:14:39,000 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a2e960de3464409b1430bb4b5735428, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1732137275588 2024-11-20T21:14:39,001 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 882d6fe22a944cfea0118c8f0bad6398, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1732137275658 2024-11-20T21:14:39,001 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 85d8701f7bf24b9ab6c4f8de75cef0d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732137276841 2024-11-20T21:14:39,009 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 917d9f6cd1adb0d5f4ed70d067a87108#C#compaction#161 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:39,009 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/1c4bfb3695034a848a92a6866a67b3d3 is 50, key is test_row_0/C:col10/1732137276841/Put/seqid=0 2024-11-20T21:14:39,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742020_1196 (size=12881) 2024-11-20T21:14:39,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:39,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137339017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:39,023 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 0b16ab3f13b7,36987,1732137236536 for 'test_row_2' in 917d9f6cd1adb0d5f4ed70d067a87108 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T21:14:39,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:39,023 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:39,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:39,024 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/1c4bfb3695034a848a92a6866a67b3d3 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/1c4bfb3695034a848a92a6866a67b3d3 2024-11-20T21:14:39,029 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 917d9f6cd1adb0d5f4ed70d067a87108/C of 917d9f6cd1adb0d5f4ed70d067a87108 into 1c4bfb3695034a848a92a6866a67b3d3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:39,029 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:39,029 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., storeName=917d9f6cd1adb0d5f4ed70d067a87108/C, priority=12, startTime=1732137278524; duration=0sec 2024-11-20T21:14:39,029 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:39,029 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:C 2024-11-20T21:14:39,032 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/b644816e0e084f4384c233342c1ac260 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/b644816e0e084f4384c233342c1ac260 2024-11-20T21:14:39,037 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 917d9f6cd1adb0d5f4ed70d067a87108/A of 917d9f6cd1adb0d5f4ed70d067a87108 into b644816e0e084f4384c233342c1ac260(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:39,037 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:39,037 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., storeName=917d9f6cd1adb0d5f4ed70d067a87108/A, priority=12, startTime=1732137278524; duration=0sec 2024-11-20T21:14:39,037 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:39,037 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:A 2024-11-20T21:14:39,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:39,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137339067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:39,073 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 0b16ab3f13b7,36987,1732137236536 for 'test_row_1' in 917d9f6cd1adb0d5f4ed70d067a87108 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2219 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T21:14:39,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:39,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:39,073 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:39,136 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/1e42c9dc636248398ef07fecf42a9348 2024-11-20T21:14:39,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/16007388d96047288b7787e6ccc39c7e as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/16007388d96047288b7787e6ccc39c7e 2024-11-20T21:14:39,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/16007388d96047288b7787e6ccc39c7e, entries=250, sequenceid=295, filesize=47.5 K 2024-11-20T21:14:39,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/1c8c41e0fd4446b999bb1b84baa94621 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/1c8c41e0fd4446b999bb1b84baa94621 2024-11-20T21:14:39,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/1c8c41e0fd4446b999bb1b84baa94621, entries=150, sequenceid=295, filesize=12.0 K 2024-11-20T21:14:39,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/1e42c9dc636248398ef07fecf42a9348 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/1e42c9dc636248398ef07fecf42a9348 2024-11-20T21:14:39,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/1e42c9dc636248398ef07fecf42a9348, entries=150, sequenceid=295, filesize=12.0 K 2024-11-20T21:14:39,159 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~91.69 KB/93890, heapSize ~267.14 KB/273552, currentSize=87.22 KB/89310 for 917d9f6cd1adb0d5f4ed70d067a87108 in 595ms, sequenceid=295, compaction requested=false 2024-11-20T21:14:39,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:39,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:39,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 917d9f6cd1adb0d5f4ed70d067a87108 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T21:14:39,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=A 2024-11-20T21:14:39,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:39,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=B 2024-11-20T21:14:39,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:39,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=C 2024-11-20T21:14:39,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:39,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ea821fcf661d4ccc97bef84f62580464_917d9f6cd1adb0d5f4ed70d067a87108 is 50, key is test_row_0/A:col10/1732137278623/Put/seqid=0 2024-11-20T21:14:39,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742021_1197 (size=14994) 2024-11-20T21:14:39,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:39,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137339242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:39,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:39,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:39,252 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:39,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:39,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137339257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:39,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:39,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137339258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:39,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:39,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:39,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:39,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:39,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:39,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:39,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:39,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137339357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:39,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:39,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:39,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:39,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-20T21:14:39,467 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:14:39,468 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:14:39,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-20T21:14:39,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T21:14:39,470 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:14:39,470 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:14:39,470 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:14:39,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:39,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137339567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:39,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:39,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:39,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:39,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T21:14:39,594 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:39,598 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ea821fcf661d4ccc97bef84f62580464_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ea821fcf661d4ccc97bef84f62580464_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:39,599 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/b616d5ba34884772b73bf200e0f5d27e, store: [table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:39,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/b616d5ba34884772b73bf200e0f5d27e is 175, key is test_row_0/A:col10/1732137278623/Put/seqid=0 2024-11-20T21:14:39,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742022_1198 (size=39949) 2024-11-20T21:14:39,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T21:14:39,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:39,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:39,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:39,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:39,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:39,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:39,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:39,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137339767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:39,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:39,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:39,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:39,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T21:14:39,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:39,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:39,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:39,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:39,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:39,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:39,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:39,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137339776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:39,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:39,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:39,778 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:39,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T21:14:39,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:39,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137339876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:39,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:39,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:39,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:39,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T21:14:39,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:39,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:39,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:39,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:39,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:39,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:40,007 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=315, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/b616d5ba34884772b73bf200e0f5d27e 2024-11-20T21:14:40,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/ffe9bc2c79484fdd8b3fa46b14a5adc7 is 50, key is test_row_0/B:col10/1732137278623/Put/seqid=0 2024-11-20T21:14:40,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742023_1199 (size=12301) 2024-11-20T21:14:40,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/ffe9bc2c79484fdd8b3fa46b14a5adc7 2024-11-20T21:14:40,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/c2a6b958059d47ab8bd2f79462a23f82 is 50, key is test_row_0/C:col10/1732137278623/Put/seqid=0 2024-11-20T21:14:40,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T21:14:40,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:40,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:40,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:40,082 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:40,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:40,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742024_1200 (size=12301) 2024-11-20T21:14:40,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/c2a6b958059d47ab8bd2f79462a23f82 2024-11-20T21:14:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T21:14:40,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/b616d5ba34884772b73bf200e0f5d27e as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/b616d5ba34884772b73bf200e0f5d27e 2024-11-20T21:14:40,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/b616d5ba34884772b73bf200e0f5d27e, entries=200, sequenceid=315, filesize=39.0 K 2024-11-20T21:14:40,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/ffe9bc2c79484fdd8b3fa46b14a5adc7 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/ffe9bc2c79484fdd8b3fa46b14a5adc7 2024-11-20T21:14:40,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/ffe9bc2c79484fdd8b3fa46b14a5adc7, entries=150, sequenceid=315, filesize=12.0 K 2024-11-20T21:14:40,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/c2a6b958059d47ab8bd2f79462a23f82 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/c2a6b958059d47ab8bd2f79462a23f82 2024-11-20T21:14:40,129 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/c2a6b958059d47ab8bd2f79462a23f82, entries=150, sequenceid=315, filesize=12.0 K 2024-11-20T21:14:40,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 917d9f6cd1adb0d5f4ed70d067a87108 in 950ms, sequenceid=315, compaction requested=true 2024-11-20T21:14:40,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:40,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:14:40,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:40,131 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:40,131 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:40,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:14:40,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:40,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:14:40,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:40,132 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120423 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:40,132 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 917d9f6cd1adb0d5f4ed70d067a87108/A is initiating minor compaction (all files) 2024-11-20T21:14:40,132 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 917d9f6cd1adb0d5f4ed70d067a87108/A in TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:40,132 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/b644816e0e084f4384c233342c1ac260, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/16007388d96047288b7787e6ccc39c7e, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/b616d5ba34884772b73bf200e0f5d27e] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp, totalSize=117.6 K 2024-11-20T21:14:40,132 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:40,132 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. files: [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/b644816e0e084f4384c233342c1ac260, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/16007388d96047288b7787e6ccc39c7e, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/b616d5ba34884772b73bf200e0f5d27e] 2024-11-20T21:14:40,133 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:40,133 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 917d9f6cd1adb0d5f4ed70d067a87108/B is initiating minor compaction (all files) 2024-11-20T21:14:40,133 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 917d9f6cd1adb0d5f4ed70d067a87108/B in TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:40,133 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting b644816e0e084f4384c233342c1ac260, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732137276841 2024-11-20T21:14:40,133 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/0fade51b0a324076bac7484012c068d5, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/1c8c41e0fd4446b999bb1b84baa94621, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/ffe9bc2c79484fdd8b3fa46b14a5adc7] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp, totalSize=36.6 K 2024-11-20T21:14:40,134 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16007388d96047288b7787e6ccc39c7e, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732137277908 2024-11-20T21:14:40,134 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fade51b0a324076bac7484012c068d5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732137276841 2024-11-20T21:14:40,134 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting b616d5ba34884772b73bf200e0f5d27e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732137278602 2024-11-20T21:14:40,135 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c8c41e0fd4446b999bb1b84baa94621, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732137277913 2024-11-20T21:14:40,135 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting ffe9bc2c79484fdd8b3fa46b14a5adc7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732137278602 2024-11-20T21:14:40,147 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 917d9f6cd1adb0d5f4ed70d067a87108#B#compaction#165 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:40,147 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/4c880ee597ab4d66b1c17915ecd32d4d is 50, key is test_row_0/B:col10/1732137278623/Put/seqid=0 2024-11-20T21:14:40,149 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:40,157 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120f0e142966a6a481c816773563909fa6f_917d9f6cd1adb0d5f4ed70d067a87108 store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:40,159 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120f0e142966a6a481c816773563909fa6f_917d9f6cd1adb0d5f4ed70d067a87108, store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:40,159 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f0e142966a6a481c816773563909fa6f_917d9f6cd1adb0d5f4ed70d067a87108 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:40,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742025_1201 (size=12983) 2024-11-20T21:14:40,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742026_1202 (size=4469) 2024-11-20T21:14:40,180 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 917d9f6cd1adb0d5f4ed70d067a87108#A#compaction#166 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:40,181 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/4afcf0c6dcbb4325b4fba11b94ed2590 is 175, key is test_row_0/A:col10/1732137278623/Put/seqid=0 2024-11-20T21:14:40,181 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/4c880ee597ab4d66b1c17915ecd32d4d as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/4c880ee597ab4d66b1c17915ecd32d4d 2024-11-20T21:14:40,189 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 917d9f6cd1adb0d5f4ed70d067a87108/B of 917d9f6cd1adb0d5f4ed70d067a87108 into 4c880ee597ab4d66b1c17915ecd32d4d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:40,189 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:40,189 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., storeName=917d9f6cd1adb0d5f4ed70d067a87108/B, priority=13, startTime=1732137280131; duration=0sec 2024-11-20T21:14:40,189 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:40,189 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:B 2024-11-20T21:14:40,189 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:40,191 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:40,191 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 917d9f6cd1adb0d5f4ed70d067a87108/C is initiating minor compaction (all files) 2024-11-20T21:14:40,191 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 917d9f6cd1adb0d5f4ed70d067a87108/C in TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:40,191 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/1c4bfb3695034a848a92a6866a67b3d3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/1e42c9dc636248398ef07fecf42a9348, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/c2a6b958059d47ab8bd2f79462a23f82] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp, totalSize=36.6 K 2024-11-20T21:14:40,192 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c4bfb3695034a848a92a6866a67b3d3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732137276841 2024-11-20T21:14:40,193 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e42c9dc636248398ef07fecf42a9348, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1732137277913 2024-11-20T21:14:40,193 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting c2a6b958059d47ab8bd2f79462a23f82, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732137278602 2024-11-20T21:14:40,207 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 917d9f6cd1adb0d5f4ed70d067a87108#C#compaction#167 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:40,208 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/16bd3f9e593b4ca5b289af9262533b99 is 50, key is test_row_0/C:col10/1732137278623/Put/seqid=0 2024-11-20T21:14:40,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742027_1203 (size=31937) 2024-11-20T21:14:40,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742028_1204 (size=12983) 2024-11-20T21:14:40,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-20T21:14:40,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:40,235 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 917d9f6cd1adb0d5f4ed70d067a87108 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-20T21:14:40,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=A 2024-11-20T21:14:40,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:40,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=B 2024-11-20T21:14:40,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:40,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=C 2024-11-20T21:14:40,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:40,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c6fce599a1e44b6b96ebfdd2b720edc9_917d9f6cd1adb0d5f4ed70d067a87108 is 50, key is test_row_0/A:col10/1732137279240/Put/seqid=0 2024-11-20T21:14:40,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742029_1205 (size=12454) 2024-11-20T21:14:40,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:40,391 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:40,427 INFO [master/0b16ab3f13b7:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-20T21:14:40,427 INFO [master/0b16ab3f13b7:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-20T21:14:40,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:40,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137340458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:40,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:40,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:40,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:40,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:40,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137340569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:40,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:40,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:40,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:40,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T21:14:40,630 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/4afcf0c6dcbb4325b4fba11b94ed2590 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/4afcf0c6dcbb4325b4fba11b94ed2590 2024-11-20T21:14:40,632 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/16bd3f9e593b4ca5b289af9262533b99 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/16bd3f9e593b4ca5b289af9262533b99 2024-11-20T21:14:40,647 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 917d9f6cd1adb0d5f4ed70d067a87108/A of 917d9f6cd1adb0d5f4ed70d067a87108 into 4afcf0c6dcbb4325b4fba11b94ed2590(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:40,647 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:40,647 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., storeName=917d9f6cd1adb0d5f4ed70d067a87108/A, priority=13, startTime=1732137280131; duration=0sec 2024-11-20T21:14:40,647 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:40,647 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 917d9f6cd1adb0d5f4ed70d067a87108/C of 917d9f6cd1adb0d5f4ed70d067a87108 into 16bd3f9e593b4ca5b289af9262533b99(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:40,647 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:A 2024-11-20T21:14:40,647 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:40,647 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., storeName=917d9f6cd1adb0d5f4ed70d067a87108/C, priority=13, startTime=1732137280131; duration=0sec 2024-11-20T21:14:40,648 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:40,648 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:C 2024-11-20T21:14:40,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:40,665 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c6fce599a1e44b6b96ebfdd2b720edc9_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c6fce599a1e44b6b96ebfdd2b720edc9_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:40,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/14241ce8fdba4fdc914436adacc5897c, store: [table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:40,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/14241ce8fdba4fdc914436adacc5897c is 175, key is test_row_0/A:col10/1732137279240/Put/seqid=0 2024-11-20T21:14:40,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742030_1206 (size=31255) 2024-11-20T21:14:40,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:40,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137340777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:40,778 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:40,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137340777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:40,778 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 0b16ab3f13b7,36987,1732137236536 for 'test_row_0' in 917d9f6cd1adb0d5f4ed70d067a87108 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T21:14:40,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:40,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:40,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:40,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:40,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:40,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:40,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:40,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137340797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:40,798 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 0b16ab3f13b7,36987,1732137236536 for 'test_row_0' in 917d9f6cd1adb0d5f4ed70d067a87108 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T21:14:40,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:40,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:40,799 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:41,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:41,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137341037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:41,048 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 0b16ab3f13b7,36987,1732137236536 for 'test_row_2' in 917d9f6cd1adb0d5f4ed70d067a87108 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T21:14:41,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:41,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:41,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:41,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:41,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137341089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:41,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:41,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:41,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:41,097 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=331, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/14241ce8fdba4fdc914436adacc5897c 2024-11-20T21:14:41,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:41,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137341089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:41,103 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 0b16ab3f13b7,36987,1732137236536 for 'test_row_1' in 917d9f6cd1adb0d5f4ed70d067a87108 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4250 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T21:14:41,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:41,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:41,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:41,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/2525a0a6c7d2421ebf396baf6c82ecf8 is 50, key is test_row_0/B:col10/1732137279240/Put/seqid=0 2024-11-20T21:14:41,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742031_1207 (size=12301) 2024-11-20T21:14:41,194 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/2525a0a6c7d2421ebf396baf6c82ecf8 2024-11-20T21:14:41,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/e2d402c037154bb69d02a86a2790e5cb is 50, key is test_row_0/C:col10/1732137279240/Put/seqid=0 2024-11-20T21:14:41,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742032_1208 (size=12301) 2024-11-20T21:14:41,273 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/e2d402c037154bb69d02a86a2790e5cb 2024-11-20T21:14:41,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/14241ce8fdba4fdc914436adacc5897c as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/14241ce8fdba4fdc914436adacc5897c 2024-11-20T21:14:41,295 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/14241ce8fdba4fdc914436adacc5897c, entries=150, sequenceid=331, filesize=30.5 K 2024-11-20T21:14:41,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/2525a0a6c7d2421ebf396baf6c82ecf8 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/2525a0a6c7d2421ebf396baf6c82ecf8 2024-11-20T21:14:41,302 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/2525a0a6c7d2421ebf396baf6c82ecf8, entries=150, sequenceid=331, filesize=12.0 K 2024-11-20T21:14:41,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/e2d402c037154bb69d02a86a2790e5cb as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/e2d402c037154bb69d02a86a2790e5cb 2024-11-20T21:14:41,310 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/e2d402c037154bb69d02a86a2790e5cb, entries=150, sequenceid=331, filesize=12.0 K 2024-11-20T21:14:41,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-20T21:14:41,315 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 917d9f6cd1adb0d5f4ed70d067a87108 in 1080ms, sequenceid=331, compaction requested=false 2024-11-20T21:14:41,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:41,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:41,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-20T21:14:41,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-20T21:14:41,320 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-20T21:14:41,320 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8480 sec 2024-11-20T21:14:41,326 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.8530 sec 2024-11-20T21:14:41,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:41,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-20T21:14:41,629 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:14:41,631 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:14:41,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-20T21:14:41,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 917d9f6cd1adb0d5f4ed70d067a87108 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-20T21:14:41,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=A 2024-11-20T21:14:41,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:41,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=B 2024-11-20T21:14:41,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:41,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=C 2024-11-20T21:14:41,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:41,633 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:14:41,634 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:14:41,634 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:14:41,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T21:14:41,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411204e355f4eda4342e1b8a09ed5a44c7c77_917d9f6cd1adb0d5f4ed70d067a87108 is 50, key is test_row_0/A:col10/1732137280441/Put/seqid=0 2024-11-20T21:14:41,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742033_1209 (size=14994) 2024-11-20T21:14:41,726 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:41,732 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411204e355f4eda4342e1b8a09ed5a44c7c77_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411204e355f4eda4342e1b8a09ed5a44c7c77_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:41,733 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/8a07dd0dce914027aeebb912e565360f, store: [table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:41,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/8a07dd0dce914027aeebb912e565360f is 175, key is test_row_0/A:col10/1732137280441/Put/seqid=0 2024-11-20T21:14:41,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T21:14:41,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742034_1210 (size=39949) 2024-11-20T21:14:41,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:41,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137341744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:41,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:41,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:41,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:41,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T21:14:41,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:41,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:41,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:41,789 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:41,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:41,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:41,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:41,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137341867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:41,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:41,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:41,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:41,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T21:14:41,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:41,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:41,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:41,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:41,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:41,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:41,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T21:14:42,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:42,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137342077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:42,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:42,096 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T21:14:42,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:42,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:42,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:42,100 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:42,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:42,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:42,157 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=351, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/8a07dd0dce914027aeebb912e565360f 2024-11-20T21:14:42,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/d5a21c7a6b9b4b579697ac8f17744655 is 50, key is test_row_0/B:col10/1732137280441/Put/seqid=0 2024-11-20T21:14:42,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742035_1211 (size=12301) 2024-11-20T21:14:42,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/d5a21c7a6b9b4b579697ac8f17744655 2024-11-20T21:14:42,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T21:14:42,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/86ebc7091762433d9e103edb2a8f99f1 is 50, key is test_row_0/C:col10/1732137280441/Put/seqid=0 2024-11-20T21:14:42,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:42,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:42,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:42,255 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:42,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:42,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:42,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742036_1212 (size=12301) 2024-11-20T21:14:42,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/86ebc7091762433d9e103edb2a8f99f1 2024-11-20T21:14:42,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T21:14:42,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/8a07dd0dce914027aeebb912e565360f as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/8a07dd0dce914027aeebb912e565360f 2024-11-20T21:14:42,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/8a07dd0dce914027aeebb912e565360f, entries=200, sequenceid=351, filesize=39.0 K 2024-11-20T21:14:42,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/d5a21c7a6b9b4b579697ac8f17744655 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/d5a21c7a6b9b4b579697ac8f17744655 2024-11-20T21:14:42,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/d5a21c7a6b9b4b579697ac8f17744655, entries=150, sequenceid=351, filesize=12.0 K 2024-11-20T21:14:42,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/86ebc7091762433d9e103edb2a8f99f1 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/86ebc7091762433d9e103edb2a8f99f1 2024-11-20T21:14:42,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/86ebc7091762433d9e103edb2a8f99f1, entries=150, sequenceid=351, filesize=12.0 K 2024-11-20T21:14:42,307 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 917d9f6cd1adb0d5f4ed70d067a87108 in 675ms, sequenceid=351, compaction requested=true 2024-11-20T21:14:42,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:42,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:14:42,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:42,307 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:42,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:14:42,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:42,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:14:42,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T21:14:42,307 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:42,308 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:42,308 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 917d9f6cd1adb0d5f4ed70d067a87108/A is initiating minor compaction (all files) 2024-11-20T21:14:42,309 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 917d9f6cd1adb0d5f4ed70d067a87108/A in TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:42,309 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:42,309 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/4afcf0c6dcbb4325b4fba11b94ed2590, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/14241ce8fdba4fdc914436adacc5897c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/8a07dd0dce914027aeebb912e565360f] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp, totalSize=100.7 K 2024-11-20T21:14:42,309 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 917d9f6cd1adb0d5f4ed70d067a87108/B is initiating minor compaction (all files) 2024-11-20T21:14:42,309 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:42,309 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 917d9f6cd1adb0d5f4ed70d067a87108/B in TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:42,309 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. files: [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/4afcf0c6dcbb4325b4fba11b94ed2590, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/14241ce8fdba4fdc914436adacc5897c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/8a07dd0dce914027aeebb912e565360f] 2024-11-20T21:14:42,309 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/4c880ee597ab4d66b1c17915ecd32d4d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/2525a0a6c7d2421ebf396baf6c82ecf8, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/d5a21c7a6b9b4b579697ac8f17744655] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp, totalSize=36.7 K 2024-11-20T21:14:42,309 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4afcf0c6dcbb4325b4fba11b94ed2590, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732137278602 2024-11-20T21:14:42,309 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c880ee597ab4d66b1c17915ecd32d4d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732137278602 2024-11-20T21:14:42,310 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14241ce8fdba4fdc914436adacc5897c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732137279216 2024-11-20T21:14:42,310 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 2525a0a6c7d2421ebf396baf6c82ecf8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732137279216 2024-11-20T21:14:42,310 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a07dd0dce914027aeebb912e565360f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732137280441 2024-11-20T21:14:42,310 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting d5a21c7a6b9b4b579697ac8f17744655, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732137280441 2024-11-20T21:14:42,344 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:42,347 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 917d9f6cd1adb0d5f4ed70d067a87108#B#compaction#175 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:42,348 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/f987ce5be4274b21ad06f56b6a4609db is 50, key is test_row_0/B:col10/1732137280441/Put/seqid=0 2024-11-20T21:14:42,351 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120d658ef1fe9604d2eb0465741086eb988_917d9f6cd1adb0d5f4ed70d067a87108 store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:42,353 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120d658ef1fe9604d2eb0465741086eb988_917d9f6cd1adb0d5f4ed70d067a87108, store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:42,354 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d658ef1fe9604d2eb0465741086eb988_917d9f6cd1adb0d5f4ed70d067a87108 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:42,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742037_1213 (size=13085) 2024-11-20T21:14:42,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742038_1214 (size=4469) 2024-11-20T21:14:42,394 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 917d9f6cd1adb0d5f4ed70d067a87108#A#compaction#174 average throughput is 0.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:42,395 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/2d25d282b9454559be977af979198b49 is 175, key is test_row_0/A:col10/1732137280441/Put/seqid=0 2024-11-20T21:14:42,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:42,401 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 917d9f6cd1adb0d5f4ed70d067a87108 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-20T21:14:42,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=A 2024-11-20T21:14:42,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:42,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=B 2024-11-20T21:14:42,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:42,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=C 2024-11-20T21:14:42,401 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:42,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T21:14:42,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:42,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:42,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:42,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:42,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:42,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:42,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209a6663ad9bc64646ab5bf88bfee65282_917d9f6cd1adb0d5f4ed70d067a87108 is 50, key is test_row_0/A:col10/1732137281689/Put/seqid=0 2024-11-20T21:14:42,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742039_1215 (size=32039) 2024-11-20T21:14:42,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742040_1216 (size=14994) 2024-11-20T21:14:42,451 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:42,456 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209a6663ad9bc64646ab5bf88bfee65282_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209a6663ad9bc64646ab5bf88bfee65282_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:42,459 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/60894b0cf0c84cbdafcf26c5bb2bedcd, store: [table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:42,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/60894b0cf0c84cbdafcf26c5bb2bedcd is 175, key is test_row_0/A:col10/1732137281689/Put/seqid=0 2024-11-20T21:14:42,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742041_1217 (size=39949) 2024-11-20T21:14:42,494 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=366, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/60894b0cf0c84cbdafcf26c5bb2bedcd 2024-11-20T21:14:42,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/8904667fa50e442eb19e1378d0d78fdb is 50, key is test_row_0/B:col10/1732137281689/Put/seqid=0 2024-11-20T21:14:42,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:42,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137342510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:42,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:42,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:42,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:42,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742042_1218 (size=12301) 2024-11-20T21:14:42,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/8904667fa50e442eb19e1378d0d78fdb 2024-11-20T21:14:42,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/d968c15f9b674847ab50477d79ae2cb4 is 50, key is test_row_0/C:col10/1732137281689/Put/seqid=0 2024-11-20T21:14:42,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T21:14:42,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:42,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:42,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:42,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:42,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:42,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:42,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742043_1219 (size=12301) 2024-11-20T21:14:42,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/d968c15f9b674847ab50477d79ae2cb4 2024-11-20T21:14:42,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/60894b0cf0c84cbdafcf26c5bb2bedcd as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/60894b0cf0c84cbdafcf26c5bb2bedcd 2024-11-20T21:14:42,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/60894b0cf0c84cbdafcf26c5bb2bedcd, entries=200, sequenceid=366, filesize=39.0 K 2024-11-20T21:14:42,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/8904667fa50e442eb19e1378d0d78fdb as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/8904667fa50e442eb19e1378d0d78fdb 2024-11-20T21:14:42,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/8904667fa50e442eb19e1378d0d78fdb, entries=150, sequenceid=366, filesize=12.0 K 2024-11-20T21:14:42,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/d968c15f9b674847ab50477d79ae2cb4 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/d968c15f9b674847ab50477d79ae2cb4 2024-11-20T21:14:42,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/d968c15f9b674847ab50477d79ae2cb4, entries=150, sequenceid=366, filesize=12.0 K 2024-11-20T21:14:42,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 917d9f6cd1adb0d5f4ed70d067a87108 in 207ms, sequenceid=366, compaction requested=true 2024-11-20T21:14:42,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:42,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:A, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:14:42,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T21:14:42,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:B, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:14:42,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-20T21:14:42,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:14:42,608 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-20T21:14:42,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:42,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 917d9f6cd1adb0d5f4ed70d067a87108 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-20T21:14:42,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=A 2024-11-20T21:14:42,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:42,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=B 2024-11-20T21:14:42,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:42,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=C 2024-11-20T21:14:42,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:42,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120aee4a0a231ee481baadccca4f3a3ffea_917d9f6cd1adb0d5f4ed70d067a87108 is 50, key is test_row_0/A:col10/1732137282619/Put/seqid=0 2024-11-20T21:14:42,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742044_1220 (size=14994) 2024-11-20T21:14:42,652 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:42,673 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120aee4a0a231ee481baadccca4f3a3ffea_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120aee4a0a231ee481baadccca4f3a3ffea_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:42,674 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/eb7cd2a4292c489f8e5b96d11042e7b1, store: [table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:42,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/eb7cd2a4292c489f8e5b96d11042e7b1 is 175, key is test_row_0/A:col10/1732137282619/Put/seqid=0 2024-11-20T21:14:42,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:42,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137342685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:42,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:42,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:42,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:42,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742045_1221 (size=39949) 2024-11-20T21:14:42,698 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=384, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/eb7cd2a4292c489f8e5b96d11042e7b1 2024-11-20T21:14:42,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/1ddaf799408147dca6014f71305ab33c is 50, key is test_row_0/B:col10/1732137282619/Put/seqid=0 2024-11-20T21:14:42,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742046_1222 (size=12301) 2024-11-20T21:14:42,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T21:14:42,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:42,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:42,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:42,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:42,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:42,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:42,773 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/f987ce5be4274b21ad06f56b6a4609db as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/f987ce5be4274b21ad06f56b6a4609db 2024-11-20T21:14:42,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T21:14:42,785 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 917d9f6cd1adb0d5f4ed70d067a87108/B of 917d9f6cd1adb0d5f4ed70d067a87108 into f987ce5be4274b21ad06f56b6a4609db(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:42,785 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:42,785 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., storeName=917d9f6cd1adb0d5f4ed70d067a87108/B, priority=13, startTime=1732137282307; duration=0sec 2024-11-20T21:14:42,785 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-20T21:14:42,785 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:B 2024-11-20T21:14:42,786 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-11-20T21:14:42,786 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T21:14:42,786 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T21:14:42,787 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. because compaction request was cancelled 2024-11-20T21:14:42,787 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:A 2024-11-20T21:14:42,787 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:14:42,790 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:14:42,790 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 917d9f6cd1adb0d5f4ed70d067a87108/C is initiating minor compaction (all files) 2024-11-20T21:14:42,790 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 917d9f6cd1adb0d5f4ed70d067a87108/C in TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:42,790 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/16bd3f9e593b4ca5b289af9262533b99, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/e2d402c037154bb69d02a86a2790e5cb, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/86ebc7091762433d9e103edb2a8f99f1, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/d968c15f9b674847ab50477d79ae2cb4] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp, totalSize=48.7 K 2024-11-20T21:14:42,792 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 16bd3f9e593b4ca5b289af9262533b99, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1732137278602 2024-11-20T21:14:42,792 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting e2d402c037154bb69d02a86a2790e5cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732137279216 2024-11-20T21:14:42,793 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 86ebc7091762433d9e103edb2a8f99f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732137280441 2024-11-20T21:14:42,793 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting d968c15f9b674847ab50477d79ae2cb4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732137281689 2024-11-20T21:14:42,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:42,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137342787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:42,802 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 0b16ab3f13b7,36987,1732137236536 for 'test_row_0' in 917d9f6cd1adb0d5f4ed70d067a87108 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T21:14:42,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:42,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:42,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:42,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:42,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137342797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:42,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:42,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:42,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:42,810 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 917d9f6cd1adb0d5f4ed70d067a87108#C#compaction#181 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:42,810 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/9b8de26d699f4e1d953eee3dc9b7f970 is 50, key is test_row_0/C:col10/1732137281689/Put/seqid=0 2024-11-20T21:14:42,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:42,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137342812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:42,815 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 0b16ab3f13b7,36987,1732137236536 for 'test_row_0' in 917d9f6cd1adb0d5f4ed70d067a87108 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T21:14:42,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:42,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:42,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:42,842 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/2d25d282b9454559be977af979198b49 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/2d25d282b9454559be977af979198b49 2024-11-20T21:14:42,858 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 917d9f6cd1adb0d5f4ed70d067a87108/A of 917d9f6cd1adb0d5f4ed70d067a87108 into 2d25d282b9454559be977af979198b49(size=31.3 K), total size for store is 70.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:42,858 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:42,858 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., storeName=917d9f6cd1adb0d5f4ed70d067a87108/A, priority=13, startTime=1732137282307; duration=0sec 2024-11-20T21:14:42,858 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T21:14:42,858 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:A 2024-11-20T21:14:42,858 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:B 2024-11-20T21:14:42,858 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-20T21:14:42,858 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T21:14:42,858 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T21:14:42,858 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. because compaction request was cancelled 2024-11-20T21:14:42,858 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:C 2024-11-20T21:14:42,858 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-20T21:14:42,859 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T21:14:42,859 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T21:14:42,859 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. because compaction request was cancelled 2024-11-20T21:14:42,859 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:B 2024-11-20T21:14:42,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742047_1223 (size=13119) 2024-11-20T21:14:42,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T21:14:42,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:42,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:42,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:42,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:42,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:42,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:42,907 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/9b8de26d699f4e1d953eee3dc9b7f970 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/9b8de26d699f4e1d953eee3dc9b7f970 2024-11-20T21:14:42,913 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 917d9f6cd1adb0d5f4ed70d067a87108/C of 917d9f6cd1adb0d5f4ed70d067a87108 into 9b8de26d699f4e1d953eee3dc9b7f970(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:42,913 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:42,913 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., storeName=917d9f6cd1adb0d5f4ed70d067a87108/C, priority=12, startTime=1732137282608; duration=0sec 2024-11-20T21:14:42,913 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:42,913 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:C 2024-11-20T21:14:43,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137343007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:43,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:43,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:43,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:43,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T21:14:43,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:43,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:43,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:43,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:43,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:43,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:43,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/1ddaf799408147dca6014f71305ab33c 2024-11-20T21:14:43,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/be91b99c028548849637509bddbbfdab is 50, key is test_row_0/C:col10/1732137282619/Put/seqid=0 2024-11-20T21:14:43,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742048_1224 (size=12301) 2024-11-20T21:14:43,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/be91b99c028548849637509bddbbfdab 2024-11-20T21:14:43,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T21:14:43,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:43,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:43,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:43,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:43,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:43,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:43,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/eb7cd2a4292c489f8e5b96d11042e7b1 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/eb7cd2a4292c489f8e5b96d11042e7b1 2024-11-20T21:14:43,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/eb7cd2a4292c489f8e5b96d11042e7b1, entries=200, sequenceid=384, filesize=39.0 K 2024-11-20T21:14:43,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/1ddaf799408147dca6014f71305ab33c as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/1ddaf799408147dca6014f71305ab33c 2024-11-20T21:14:43,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/1ddaf799408147dca6014f71305ab33c, entries=150, sequenceid=384, filesize=12.0 K 2024-11-20T21:14:43,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/be91b99c028548849637509bddbbfdab as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/be91b99c028548849637509bddbbfdab 2024-11-20T21:14:43,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/be91b99c028548849637509bddbbfdab, entries=150, sequenceid=384, filesize=12.0 K 2024-11-20T21:14:43,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 917d9f6cd1adb0d5f4ed70d067a87108 in 596ms, sequenceid=384, compaction requested=true 2024-11-20T21:14:43,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:43,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:14:43,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:43,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:14:43,219 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:43,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:43,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:14:43,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T21:14:43,220 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111937 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:43,220 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 917d9f6cd1adb0d5f4ed70d067a87108/A is initiating minor compaction (all files) 2024-11-20T21:14:43,220 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 917d9f6cd1adb0d5f4ed70d067a87108/A in TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:43,220 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/2d25d282b9454559be977af979198b49, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/60894b0cf0c84cbdafcf26c5bb2bedcd, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/eb7cd2a4292c489f8e5b96d11042e7b1] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp, totalSize=109.3 K 2024-11-20T21:14:43,220 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:43,220 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. files: [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/2d25d282b9454559be977af979198b49, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/60894b0cf0c84cbdafcf26c5bb2bedcd, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/eb7cd2a4292c489f8e5b96d11042e7b1] 2024-11-20T21:14:43,221 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d25d282b9454559be977af979198b49, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732137280441 2024-11-20T21:14:43,221 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:43,222 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60894b0cf0c84cbdafcf26c5bb2bedcd, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732137281689 2024-11-20T21:14:43,222 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb7cd2a4292c489f8e5b96d11042e7b1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732137282445 2024-11-20T21:14:43,223 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:43,223 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 917d9f6cd1adb0d5f4ed70d067a87108/B is initiating minor compaction (all files) 2024-11-20T21:14:43,223 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 917d9f6cd1adb0d5f4ed70d067a87108/B in TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:43,223 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/f987ce5be4274b21ad06f56b6a4609db, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/8904667fa50e442eb19e1378d0d78fdb, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/1ddaf799408147dca6014f71305ab33c] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp, totalSize=36.8 K 2024-11-20T21:14:43,224 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting f987ce5be4274b21ad06f56b6a4609db, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1732137280441 2024-11-20T21:14:43,224 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8904667fa50e442eb19e1378d0d78fdb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732137281689 2024-11-20T21:14:43,225 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ddaf799408147dca6014f71305ab33c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732137282482 2024-11-20T21:14:43,237 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:43,240 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120b812378649224be8bdae3692fbfbe2b9_917d9f6cd1adb0d5f4ed70d067a87108 store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:43,242 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120b812378649224be8bdae3692fbfbe2b9_917d9f6cd1adb0d5f4ed70d067a87108, store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:43,243 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b812378649224be8bdae3692fbfbe2b9_917d9f6cd1adb0d5f4ed70d067a87108 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:43,251 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 917d9f6cd1adb0d5f4ed70d067a87108#B#compaction#184 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:43,252 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/cbc830a87d9c46b7bfeba96dcaab7f3c is 50, key is test_row_0/B:col10/1732137282619/Put/seqid=0 2024-11-20T21:14:43,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742049_1225 (size=4469) 2024-11-20T21:14:43,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742050_1226 (size=13187) 2024-11-20T21:14:43,317 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/cbc830a87d9c46b7bfeba96dcaab7f3c as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/cbc830a87d9c46b7bfeba96dcaab7f3c 2024-11-20T21:14:43,326 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 917d9f6cd1adb0d5f4ed70d067a87108/B of 917d9f6cd1adb0d5f4ed70d067a87108 into cbc830a87d9c46b7bfeba96dcaab7f3c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:43,326 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:43,326 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., storeName=917d9f6cd1adb0d5f4ed70d067a87108/B, priority=13, startTime=1732137283219; duration=0sec 2024-11-20T21:14:43,326 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:43,326 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:B 2024-11-20T21:14:43,326 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-20T21:14:43,327 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T21:14:43,327 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T21:14:43,327 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. because compaction request was cancelled 2024-11-20T21:14:43,327 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:C 2024-11-20T21:14:43,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:43,338 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 917d9f6cd1adb0d5f4ed70d067a87108 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-20T21:14:43,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=A 2024-11-20T21:14:43,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:43,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=B 2024-11-20T21:14:43,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:43,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=C 2024-11-20T21:14:43,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:43,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T21:14:43,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:43,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:43,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:43,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:43,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:43,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:43,360 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112089404cbe7c6f4d4bba6c6c2c7c314ebd_917d9f6cd1adb0d5f4ed70d067a87108 is 50, key is test_row_0/A:col10/1732137283335/Put/seqid=0 2024-11-20T21:14:43,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742051_1227 (size=14994) 2024-11-20T21:14:43,433 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:43,438 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112089404cbe7c6f4d4bba6c6c2c7c314ebd_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112089404cbe7c6f4d4bba6c6c2c7c314ebd_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:43,439 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/3477dc4643ca41298183fa8f367b9be2, store: [table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:43,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/3477dc4643ca41298183fa8f367b9be2 is 175, key is test_row_0/A:col10/1732137283335/Put/seqid=0 2024-11-20T21:14:43,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742052_1228 (size=39949) 2024-11-20T21:14:43,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T21:14:43,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:43,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:43,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:43,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:43,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:43,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:43,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:43,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137343516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:43,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:43,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:43,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:43,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137343641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:43,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T21:14:43,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:43,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:43,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:43,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:43,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:43,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:43,686 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 917d9f6cd1adb0d5f4ed70d067a87108#A#compaction#183 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:43,687 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/8258ceac06ea4e0da9753744956e5d41 is 175, key is test_row_0/A:col10/1732137282619/Put/seqid=0 2024-11-20T21:14:43,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742053_1229 (size=32141) 2024-11-20T21:14:43,735 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/8258ceac06ea4e0da9753744956e5d41 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/8258ceac06ea4e0da9753744956e5d41 2024-11-20T21:14:43,742 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 917d9f6cd1adb0d5f4ed70d067a87108/A of 917d9f6cd1adb0d5f4ed70d067a87108 into 8258ceac06ea4e0da9753744956e5d41(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:43,742 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:43,742 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., storeName=917d9f6cd1adb0d5f4ed70d067a87108/A, priority=13, startTime=1732137283218; duration=0sec 2024-11-20T21:14:43,742 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:43,742 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:A 2024-11-20T21:14:43,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T21:14:43,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T21:14:43,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:43,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:43,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:43,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:43,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:43,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:43,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:43,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137343867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:43,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:43,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:43,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:43,880 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=403, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/3477dc4643ca41298183fa8f367b9be2 2024-11-20T21:14:43,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/9ca5a944b32b4c8cbc6a3e6e236ce4fa is 50, key is test_row_0/B:col10/1732137283335/Put/seqid=0 2024-11-20T21:14:43,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742054_1230 (size=12301) 2024-11-20T21:14:43,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T21:14:43,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:43,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:43,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:43,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:43,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:43,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:44,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T21:14:44,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:44,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:44,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:44,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:44,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:44,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:44,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:44,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137344178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:44,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:44,187 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:44,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:44,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T21:14:44,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:44,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:44,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:44,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:44,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:44,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:44,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/9ca5a944b32b4c8cbc6a3e6e236ce4fa 2024-11-20T21:14:44,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/59c4e56acc88417ba933f68621a184b3 is 50, key is test_row_0/C:col10/1732137283335/Put/seqid=0 2024-11-20T21:14:44,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742055_1231 (size=12301) 2024-11-20T21:14:44,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/59c4e56acc88417ba933f68621a184b3 2024-11-20T21:14:44,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/3477dc4643ca41298183fa8f367b9be2 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/3477dc4643ca41298183fa8f367b9be2 2024-11-20T21:14:44,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/3477dc4643ca41298183fa8f367b9be2, entries=200, sequenceid=403, filesize=39.0 K 2024-11-20T21:14:44,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/9ca5a944b32b4c8cbc6a3e6e236ce4fa as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/9ca5a944b32b4c8cbc6a3e6e236ce4fa 2024-11-20T21:14:44,396 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/9ca5a944b32b4c8cbc6a3e6e236ce4fa, entries=150, sequenceid=403, filesize=12.0 K 2024-11-20T21:14:44,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/59c4e56acc88417ba933f68621a184b3 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/59c4e56acc88417ba933f68621a184b3 2024-11-20T21:14:44,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/59c4e56acc88417ba933f68621a184b3, entries=150, sequenceid=403, filesize=12.0 K 2024-11-20T21:14:44,406 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 917d9f6cd1adb0d5f4ed70d067a87108 in 1069ms, sequenceid=403, compaction requested=true 2024-11-20T21:14:44,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:44,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:14:44,406 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:44,406 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-20T21:14:44,407 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-20T21:14:44,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:14:44,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:44,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:14:44,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:44,407 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T21:14:44,407 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T21:14:44,407 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. because compaction request was cancelled 2024-11-20T21:14:44,407 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:A 2024-11-20T21:14:44,407 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:44,408 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T21:14:44,408 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T21:14:44,408 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. because compaction request was cancelled 2024-11-20T21:14:44,408 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:B 2024-11-20T21:14:44,409 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:44,409 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 917d9f6cd1adb0d5f4ed70d067a87108/C is initiating minor compaction (all files) 2024-11-20T21:14:44,409 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 917d9f6cd1adb0d5f4ed70d067a87108/C in TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:44,409 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/9b8de26d699f4e1d953eee3dc9b7f970, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/be91b99c028548849637509bddbbfdab, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/59c4e56acc88417ba933f68621a184b3] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp, totalSize=36.8 K 2024-11-20T21:14:44,409 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b8de26d699f4e1d953eee3dc9b7f970, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1732137281689 2024-11-20T21:14:44,410 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting be91b99c028548849637509bddbbfdab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732137282482 2024-11-20T21:14:44,410 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59c4e56acc88417ba933f68621a184b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1732137282676 2024-11-20T21:14:44,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-20T21:14:44,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:44,429 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 917d9f6cd1adb0d5f4ed70d067a87108 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T21:14:44,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=A 2024-11-20T21:14:44,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:44,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=B 2024-11-20T21:14:44,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:44,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=C 2024-11-20T21:14:44,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:44,430 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 917d9f6cd1adb0d5f4ed70d067a87108#C#compaction#188 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:44,431 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/8cedc98d4c9248dbb1d4bcfbc43078f5 is 50, key is test_row_0/C:col10/1732137283335/Put/seqid=0 2024-11-20T21:14:44,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742056_1232 (size=13221) 2024-11-20T21:14:44,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120dfa213983bd748ab8984d9af14e4f877_917d9f6cd1adb0d5f4ed70d067a87108 is 50, key is test_row_0/A:col10/1732137283514/Put/seqid=0 2024-11-20T21:14:44,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742057_1233 (size=12454) 2024-11-20T21:14:44,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:44,515 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120dfa213983bd748ab8984d9af14e4f877_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120dfa213983bd748ab8984d9af14e4f877_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:44,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/0e5c8a518a784128ba4a8c1383b347b3, store: [table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:44,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/0e5c8a518a784128ba4a8c1383b347b3 is 175, key is test_row_0/A:col10/1732137283514/Put/seqid=0 2024-11-20T21:14:44,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742058_1234 (size=31255) 2024-11-20T21:14:44,533 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=421, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/0e5c8a518a784128ba4a8c1383b347b3 2024-11-20T21:14:44,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/fd50d4388ef9414f832fd4ed83a8e9e5 is 50, key is test_row_0/B:col10/1732137283514/Put/seqid=0 2024-11-20T21:14:44,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742059_1235 (size=12301) 2024-11-20T21:14:44,561 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/fd50d4388ef9414f832fd4ed83a8e9e5 2024-11-20T21:14:44,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/17cd706b17e34452af12fe92944b8c5e is 50, key is test_row_0/C:col10/1732137283514/Put/seqid=0 2024-11-20T21:14:44,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742060_1236 (size=12301) 2024-11-20T21:14:44,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:44,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:44,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:44,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137344815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:44,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:44,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:44,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:44,872 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/8cedc98d4c9248dbb1d4bcfbc43078f5 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/8cedc98d4c9248dbb1d4bcfbc43078f5 2024-11-20T21:14:44,887 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 917d9f6cd1adb0d5f4ed70d067a87108/C of 917d9f6cd1adb0d5f4ed70d067a87108 into 8cedc98d4c9248dbb1d4bcfbc43078f5(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:44,887 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:44,887 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., storeName=917d9f6cd1adb0d5f4ed70d067a87108/C, priority=13, startTime=1732137284407; duration=0sec 2024-11-20T21:14:44,887 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:44,887 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:C 2024-11-20T21:14:44,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:44,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137344927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:44,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:44,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:44,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:45,014 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/17cd706b17e34452af12fe92944b8c5e 2024-11-20T21:14:45,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/0e5c8a518a784128ba4a8c1383b347b3 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/0e5c8a518a784128ba4a8c1383b347b3 2024-11-20T21:14:45,037 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/0e5c8a518a784128ba4a8c1383b347b3, entries=150, sequenceid=421, filesize=30.5 K 2024-11-20T21:14:45,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/fd50d4388ef9414f832fd4ed83a8e9e5 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/fd50d4388ef9414f832fd4ed83a8e9e5 2024-11-20T21:14:45,043 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/fd50d4388ef9414f832fd4ed83a8e9e5, entries=150, sequenceid=421, filesize=12.0 K 2024-11-20T21:14:45,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/17cd706b17e34452af12fe92944b8c5e as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/17cd706b17e34452af12fe92944b8c5e 2024-11-20T21:14:45,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,057 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/17cd706b17e34452af12fe92944b8c5e, entries=150, sequenceid=421, filesize=12.0 K 2024-11-20T21:14:45,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,058 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 917d9f6cd1adb0d5f4ed70d067a87108 in 629ms, sequenceid=421, compaction requested=true 2024-11-20T21:14:45,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:45,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:45,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-20T21:14:45,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-20T21:14:45,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,062 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-20T21:14:45,062 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4260 sec 2024-11-20T21:14:45,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,065 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 3.4320 sec 2024-11-20T21:14:45,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:45,072 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 917d9f6cd1adb0d5f4ed70d067a87108 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T21:14:45,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=A 2024-11-20T21:14:45,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:45,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=B 2024-11-20T21:14:45,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:45,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=C 2024-11-20T21:14:45,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:45,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f88e86d74d764ff8a86c1dc2dfd1a99b_917d9f6cd1adb0d5f4ed70d067a87108 is 50, key is test_row_0/A:col10/1732137284813/Put/seqid=0 2024-11-20T21:14:45,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742061_1237 (size=12454) 2024-11-20T21:14:45,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,133 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:45,149 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f88e86d74d764ff8a86c1dc2dfd1a99b_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f88e86d74d764ff8a86c1dc2dfd1a99b_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:45,151 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/b622be437475486cb8f295be57d1c19f, store: [table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:45,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/b622be437475486cb8f295be57d1c19f is 175, key is test_row_0/A:col10/1732137284813/Put/seqid=0 2024-11-20T21:14:45,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:45,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137345157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:45,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:45,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:45,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:45,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:45,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137345161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:45,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:45,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137345161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:45,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:45,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:45,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:45,173 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:45,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:45,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:45,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742062_1238 (size=31255) 2024-11-20T21:14:45,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:45,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137345267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:45,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:45,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137345279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:45,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:45,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137345279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:45,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:45,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:45,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137345477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:45,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:45,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:45,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:45,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:45,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137345487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:45,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:45,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:45,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:45,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:45,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137345489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:45,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:45,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:45,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:45,605 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=438, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/b622be437475486cb8f295be57d1c19f 2024-11-20T21:14:45,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/a72012745fa7422a9ac29a037a5ed076 is 50, key is test_row_0/B:col10/1732137284813/Put/seqid=0 2024-11-20T21:14:45,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742063_1239 (size=12301) 2024-11-20T21:14:45,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:45,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137345797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:45,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:45,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:45,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:45,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137345797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:45,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:45,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:45,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:45,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:45,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137345797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:45,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:45,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:45,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:45,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-20T21:14:45,818 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:14:45,818 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:14:45,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-20T21:14:45,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T21:14:45,826 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:14:45,826 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:14:45,827 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:14:45,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T21:14:45,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-20T21:14:45,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:45,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:45,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:45,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:45,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:45,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:46,081 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/a72012745fa7422a9ac29a037a5ed076 2024-11-20T21:14:46,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/35f09cd1db4e45c3ad096a338a87e46f is 50, key is test_row_0/C:col10/1732137284813/Put/seqid=0 2024-11-20T21:14:46,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-20T21:14:46,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:46,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:46,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:46,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:46,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:46,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:14:46,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T21:14:46,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742064_1240 (size=12301) 2024-11-20T21:14:46,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/35f09cd1db4e45c3ad096a338a87e46f 2024-11-20T21:14:46,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/b622be437475486cb8f295be57d1c19f as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/b622be437475486cb8f295be57d1c19f 2024-11-20T21:14:46,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/b622be437475486cb8f295be57d1c19f, entries=150, sequenceid=438, filesize=30.5 K 2024-11-20T21:14:46,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/a72012745fa7422a9ac29a037a5ed076 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/a72012745fa7422a9ac29a037a5ed076 2024-11-20T21:14:46,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/a72012745fa7422a9ac29a037a5ed076, entries=150, sequenceid=438, filesize=12.0 K 2024-11-20T21:14:46,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/35f09cd1db4e45c3ad096a338a87e46f as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/35f09cd1db4e45c3ad096a338a87e46f 2024-11-20T21:14:46,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/35f09cd1db4e45c3ad096a338a87e46f, entries=150, sequenceid=438, filesize=12.0 K 2024-11-20T21:14:46,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 917d9f6cd1adb0d5f4ed70d067a87108 in 1173ms, sequenceid=438, compaction requested=true 2024-11-20T21:14:46,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:46,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:14:46,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:46,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:14:46,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T21:14:46,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:14:46,245 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-20T21:14:46,246 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:14:46,248 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:14:46,251 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134600 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:14:46,251 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 917d9f6cd1adb0d5f4ed70d067a87108/A is initiating minor compaction (all files) 2024-11-20T21:14:46,251 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 917d9f6cd1adb0d5f4ed70d067a87108/A in TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:46,251 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/8258ceac06ea4e0da9753744956e5d41, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/3477dc4643ca41298183fa8f367b9be2, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/0e5c8a518a784128ba4a8c1383b347b3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/b622be437475486cb8f295be57d1c19f] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp, totalSize=131.4 K 2024-11-20T21:14:46,251 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:46,251 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. files: [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/8258ceac06ea4e0da9753744956e5d41, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/3477dc4643ca41298183fa8f367b9be2, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/0e5c8a518a784128ba4a8c1383b347b3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/b622be437475486cb8f295be57d1c19f] 2024-11-20T21:14:46,252 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8258ceac06ea4e0da9753744956e5d41, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732137282482 2024-11-20T21:14:46,252 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3477dc4643ca41298183fa8f367b9be2, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1732137282664 2024-11-20T21:14:46,254 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e5c8a518a784128ba4a8c1383b347b3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1732137283499 2024-11-20T21:14:46,254 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting b622be437475486cb8f295be57d1c19f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732137284801 2024-11-20T21:14:46,258 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:14:46,258 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 917d9f6cd1adb0d5f4ed70d067a87108/B is initiating minor compaction (all files) 2024-11-20T21:14:46,258 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 917d9f6cd1adb0d5f4ed70d067a87108/B in TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:46,258 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/cbc830a87d9c46b7bfeba96dcaab7f3c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/9ca5a944b32b4c8cbc6a3e6e236ce4fa, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/fd50d4388ef9414f832fd4ed83a8e9e5, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/a72012745fa7422a9ac29a037a5ed076] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp, totalSize=48.9 K 2024-11-20T21:14:46,259 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting cbc830a87d9c46b7bfeba96dcaab7f3c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732137282482 2024-11-20T21:14:46,259 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ca5a944b32b4c8cbc6a3e6e236ce4fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1732137282676 2024-11-20T21:14:46,260 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting fd50d4388ef9414f832fd4ed83a8e9e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1732137283499 2024-11-20T21:14:46,261 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting a72012745fa7422a9ac29a037a5ed076, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732137284801 2024-11-20T21:14:46,290 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:46,295 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 917d9f6cd1adb0d5f4ed70d067a87108#B#compaction#196 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:46,296 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/3ca443a1b1044673ab91a81423e4456b is 50, key is test_row_0/B:col10/1732137284813/Put/seqid=0 2024-11-20T21:14:46,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-20T21:14:46,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:46,301 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 917d9f6cd1adb0d5f4ed70d067a87108 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T21:14:46,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=A 2024-11-20T21:14:46,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:46,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=B 2024-11-20T21:14:46,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:46,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=C 2024-11-20T21:14:46,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:46,298 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120bbe36d9f3cf147379b2268cdb0ba259e_917d9f6cd1adb0d5f4ed70d067a87108 store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:46,306 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120bbe36d9f3cf147379b2268cdb0ba259e_917d9f6cd1adb0d5f4ed70d067a87108, store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:46,306 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120bbe36d9f3cf147379b2268cdb0ba259e_917d9f6cd1adb0d5f4ed70d067a87108 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:46,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:46,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. as already flushing 2024-11-20T21:14:46,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742065_1241 (size=13323) 2024-11-20T21:14:46,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742066_1242 (size=4469) 2024-11-20T21:14:46,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201887a5641bc64b518f5455a9120a43cf_917d9f6cd1adb0d5f4ed70d067a87108 is 50, key is test_row_0/A:col10/1732137285148/Put/seqid=0 2024-11-20T21:14:46,388 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 917d9f6cd1adb0d5f4ed70d067a87108#A#compaction#195 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:46,388 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/130daac1a3bf4ff4b0086ebce89e639f is 175, key is test_row_0/A:col10/1732137284813/Put/seqid=0 2024-11-20T21:14:46,390 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/3ca443a1b1044673ab91a81423e4456b as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/3ca443a1b1044673ab91a81423e4456b 2024-11-20T21:14:46,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:46,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137346387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:46,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:46,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137346392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:46,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:46,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:46,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:46,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:46,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:46,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:46,402 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 917d9f6cd1adb0d5f4ed70d067a87108/B of 917d9f6cd1adb0d5f4ed70d067a87108 into 3ca443a1b1044673ab91a81423e4456b(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:46,402 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:46,402 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., storeName=917d9f6cd1adb0d5f4ed70d067a87108/B, priority=12, startTime=1732137286245; duration=0sec 2024-11-20T21:14:46,403 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:46,403 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:B 2024-11-20T21:14:46,403 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:46,404 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:46,404 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 917d9f6cd1adb0d5f4ed70d067a87108/C is initiating minor compaction (all files) 2024-11-20T21:14:46,404 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 917d9f6cd1adb0d5f4ed70d067a87108/C in TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:46,404 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/8cedc98d4c9248dbb1d4bcfbc43078f5, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/17cd706b17e34452af12fe92944b8c5e, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/35f09cd1db4e45c3ad096a338a87e46f] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp, totalSize=36.9 K 2024-11-20T21:14:46,404 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cedc98d4c9248dbb1d4bcfbc43078f5, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1732137282676 2024-11-20T21:14:46,405 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 17cd706b17e34452af12fe92944b8c5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1732137283499 2024-11-20T21:14:46,405 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 35f09cd1db4e45c3ad096a338a87e46f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732137284801 2024-11-20T21:14:46,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:46,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137346401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:46,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:46,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:46,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:46,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742068_1244 (size=32277) 2024-11-20T21:14:46,422 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 917d9f6cd1adb0d5f4ed70d067a87108#C#compaction#198 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:46,423 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/4001b324cf2548bc86415345e2346110 is 50, key is test_row_0/C:col10/1732137284813/Put/seqid=0 2024-11-20T21:14:46,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742067_1243 (size=12454) 2024-11-20T21:14:46,428 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/130daac1a3bf4ff4b0086ebce89e639f as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/130daac1a3bf4ff4b0086ebce89e639f 2024-11-20T21:14:46,434 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 917d9f6cd1adb0d5f4ed70d067a87108/A of 917d9f6cd1adb0d5f4ed70d067a87108 into 130daac1a3bf4ff4b0086ebce89e639f(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:46,434 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:46,434 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., storeName=917d9f6cd1adb0d5f4ed70d067a87108/A, priority=12, startTime=1732137286245; duration=0sec 2024-11-20T21:14:46,434 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:46,434 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:A 2024-11-20T21:14:46,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742069_1245 (size=13323) 2024-11-20T21:14:46,445 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/4001b324cf2548bc86415345e2346110 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/4001b324cf2548bc86415345e2346110 2024-11-20T21:14:46,454 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 917d9f6cd1adb0d5f4ed70d067a87108/C of 917d9f6cd1adb0d5f4ed70d067a87108 into 4001b324cf2548bc86415345e2346110(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:46,454 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:46,454 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., storeName=917d9f6cd1adb0d5f4ed70d067a87108/C, priority=13, startTime=1732137286245; duration=0sec 2024-11-20T21:14:46,454 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:46,454 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:C 2024-11-20T21:14:46,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T21:14:46,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:46,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137346514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:46,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137346514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:46,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:46,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:46,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:46,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:46,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:46,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:46,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137346517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:46,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:46,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:46,536 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:46,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:46,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137346727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:46,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:46,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:46,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137346738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:46,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:46,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:46,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:46,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:46,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137346747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:46,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:46,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:46,750 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:46,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:46,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:46,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41310 deadline: 1732137346821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:46,826 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 0b16ab3f13b7,36987,1732137236536 for 'test_row_0' in 917d9f6cd1adb0d5f4ed70d067a87108 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T21:14:46,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:46,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:46,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:46,838 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201887a5641bc64b518f5455a9120a43cf_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201887a5641bc64b518f5455a9120a43cf_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:46,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/75c27e017b9f429c8b98b5e51ac91f64, store: [table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:46,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/75c27e017b9f429c8b98b5e51ac91f64 is 175, key is test_row_0/A:col10/1732137285148/Put/seqid=0 2024-11-20T21:14:46,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:46,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41252 deadline: 1732137346857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:46,871 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 0b16ab3f13b7,36987,1732137236536 for 'test_row_0' in 917d9f6cd1adb0d5f4ed70d067a87108 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T21:14:46,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:46,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:46,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:46,880 INFO [Thread-624 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-624. 2024-11-20T21:14:46,881 DEBUG [Thread-624 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:14:46,881 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:46,881 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:46,881 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:14:46,884 INFO [Thread-620 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-620. 2024-11-20T21:14:46,884 DEBUG [Thread-620 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:14:46,884 DEBUG [Thread-620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:46,884 DEBUG [Thread-620 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:46,887 INFO [Thread-622 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-622. 2024-11-20T21:14:46,887 DEBUG [Thread-622 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:14:46,887 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:46,887 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:46,887 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:14:46,887 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:14:46,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742070_1246 (size=31255) 2024-11-20T21:14:46,891 INFO [Thread-626 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-626. 2024-11-20T21:14:46,891 DEBUG [Thread-626 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:14:46,891 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:46,891 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:46,891 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:14:46,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T21:14:47,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:47,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137347038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:47,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:47,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:47,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:47,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:47,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137347047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:47,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:47,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:47,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:47,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:47,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137347057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:47,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:47,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:47,058 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:47,286 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=455, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/75c27e017b9f429c8b98b5e51ac91f64 2024-11-20T21:14:47,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/49df26a70bd04f4eab43214fd3ff634b is 50, key is test_row_0/B:col10/1732137285148/Put/seqid=0 2024-11-20T21:14:47,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742071_1247 (size=12301) 2024-11-20T21:14:47,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41242 deadline: 1732137347547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:47,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:47,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:47,548 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:47,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:47,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41268 deadline: 1732137347556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:47,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:47,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:14:47,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:41300 deadline: 1732137347567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:47,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:47,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:14:47,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:14:47,707 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/49df26a70bd04f4eab43214fd3ff634b 2024-11-20T21:14:47,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/8a21a8e9974a4e2287c08530d3962808 is 50, key is test_row_0/C:col10/1732137285148/Put/seqid=0 2024-11-20T21:14:47,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742072_1248 (size=12301) 2024-11-20T21:14:47,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T21:14:48,120 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=455 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/8a21a8e9974a4e2287c08530d3962808 2024-11-20T21:14:48,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/75c27e017b9f429c8b98b5e51ac91f64 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/75c27e017b9f429c8b98b5e51ac91f64 2024-11-20T21:14:48,128 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/75c27e017b9f429c8b98b5e51ac91f64, entries=150, sequenceid=455, filesize=30.5 K 2024-11-20T21:14:48,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/49df26a70bd04f4eab43214fd3ff634b as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/49df26a70bd04f4eab43214fd3ff634b 2024-11-20T21:14:48,132 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/49df26a70bd04f4eab43214fd3ff634b, entries=150, sequenceid=455, filesize=12.0 K 2024-11-20T21:14:48,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/8a21a8e9974a4e2287c08530d3962808 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/8a21a8e9974a4e2287c08530d3962808 2024-11-20T21:14:48,136 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/8a21a8e9974a4e2287c08530d3962808, entries=150, sequenceid=455, filesize=12.0 K 2024-11-20T21:14:48,136 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 917d9f6cd1adb0d5f4ed70d067a87108 in 1835ms, sequenceid=455, compaction requested=false 2024-11-20T21:14:48,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:48,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:48,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-20T21:14:48,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-20T21:14:48,139 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-20T21:14:48,139 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3120 sec 2024-11-20T21:14:48,140 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.3210 sec 2024-11-20T21:14:48,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:48,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 917d9f6cd1adb0d5f4ed70d067a87108 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T21:14:48,559 INFO [Thread-615 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-615. 2024-11-20T21:14:48,559 DEBUG [Thread-615 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:14:48,559 DEBUG [Thread-615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:48,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=A 2024-11-20T21:14:48,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:48,560 DEBUG [Thread-615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:48,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=B 2024-11-20T21:14:48,560 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:14:48,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:48,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=C 2024-11-20T21:14:48,560 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:48,569 INFO [Thread-617 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-617. 2024-11-20T21:14:48,569 DEBUG [Thread-617 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:14:48,569 DEBUG [Thread-617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:48,570 DEBUG [Thread-617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:48,570 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:14:48,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200ddc92c7c3054e59859ef161d98dc46b_917d9f6cd1adb0d5f4ed70d067a87108 is 50, key is test_row_0/A:col10/1732137288557/Put/seqid=0 2024-11-20T21:14:48,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742073_1249 (size=12454) 2024-11-20T21:14:48,578 INFO [Thread-611 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-611. 2024-11-20T21:14:48,578 DEBUG [Thread-611 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:14:48,578 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:48,578 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:48,578 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:14:48,978 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:48,982 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200ddc92c7c3054e59859ef161d98dc46b_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200ddc92c7c3054e59859ef161d98dc46b_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:48,983 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/9932934ba0034ec3a1caa7c1e4c71b7f, store: [table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:48,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/9932934ba0034ec3a1caa7c1e4c71b7f is 175, key is test_row_0/A:col10/1732137288557/Put/seqid=0 2024-11-20T21:14:48,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742074_1250 (size=31255) 2024-11-20T21:14:49,391 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=474, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/9932934ba0034ec3a1caa7c1e4c71b7f 2024-11-20T21:14:49,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/abcd3679d84e4d60882bcbfd0bbc41e2 is 50, key is test_row_0/B:col10/1732137288557/Put/seqid=0 2024-11-20T21:14:49,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742075_1251 (size=12301) 2024-11-20T21:14:49,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/abcd3679d84e4d60882bcbfd0bbc41e2 2024-11-20T21:14:49,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/0b707b541b0d4f1c9fc632911befb716 is 50, key is test_row_0/C:col10/1732137288557/Put/seqid=0 2024-11-20T21:14:49,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742076_1252 (size=12301) 2024-11-20T21:14:49,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-20T21:14:49,998 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:14:50,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/0b707b541b0d4f1c9fc632911befb716 2024-11-20T21:14:50,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/9932934ba0034ec3a1caa7c1e4c71b7f as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/9932934ba0034ec3a1caa7c1e4c71b7f 2024-11-20T21:14:50,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/9932934ba0034ec3a1caa7c1e4c71b7f, entries=150, sequenceid=474, filesize=30.5 K 2024-11-20T21:14:50,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/abcd3679d84e4d60882bcbfd0bbc41e2 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/abcd3679d84e4d60882bcbfd0bbc41e2 2024-11-20T21:14:50,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/abcd3679d84e4d60882bcbfd0bbc41e2, entries=150, sequenceid=474, filesize=12.0 K 2024-11-20T21:14:50,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/0b707b541b0d4f1c9fc632911befb716 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/0b707b541b0d4f1c9fc632911befb716 2024-11-20T21:14:50,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/0b707b541b0d4f1c9fc632911befb716, entries=150, sequenceid=474, filesize=12.0 K 2024-11-20T21:14:50,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=13.42 KB/13740 for 917d9f6cd1adb0d5f4ed70d067a87108 in 1675ms, sequenceid=474, compaction requested=true 2024-11-20T21:14:50,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:50,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:14:50,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:50,235 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:50,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:14:50,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:50,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 917d9f6cd1adb0d5f4ed70d067a87108:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:14:50,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:50,235 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:50,236 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94787 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:50,236 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 917d9f6cd1adb0d5f4ed70d067a87108/A is initiating minor compaction (all files) 2024-11-20T21:14:50,236 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 917d9f6cd1adb0d5f4ed70d067a87108/A in TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:50,236 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:50,236 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/130daac1a3bf4ff4b0086ebce89e639f, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/75c27e017b9f429c8b98b5e51ac91f64, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/9932934ba0034ec3a1caa7c1e4c71b7f] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp, totalSize=92.6 K 2024-11-20T21:14:50,236 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 917d9f6cd1adb0d5f4ed70d067a87108/B is initiating minor compaction (all files) 2024-11-20T21:14:50,236 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:50,236 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. files: [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/130daac1a3bf4ff4b0086ebce89e639f, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/75c27e017b9f429c8b98b5e51ac91f64, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/9932934ba0034ec3a1caa7c1e4c71b7f] 2024-11-20T21:14:50,236 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 917d9f6cd1adb0d5f4ed70d067a87108/B in TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:50,236 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/3ca443a1b1044673ab91a81423e4456b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/49df26a70bd04f4eab43214fd3ff634b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/abcd3679d84e4d60882bcbfd0bbc41e2] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp, totalSize=37.0 K 2024-11-20T21:14:50,237 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 130daac1a3bf4ff4b0086ebce89e639f, keycount=150, bloomtype=ROW, size=31.5 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732137284801 2024-11-20T21:14:50,237 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ca443a1b1044673ab91a81423e4456b, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732137284801 2024-11-20T21:14:50,237 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75c27e017b9f429c8b98b5e51ac91f64, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1732137285129 2024-11-20T21:14:50,237 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 49df26a70bd04f4eab43214fd3ff634b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1732137285129 2024-11-20T21:14:50,238 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9932934ba0034ec3a1caa7c1e4c71b7f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1732137286368 2024-11-20T21:14:50,238 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting abcd3679d84e4d60882bcbfd0bbc41e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1732137286368 2024-11-20T21:14:50,247 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 917d9f6cd1adb0d5f4ed70d067a87108#B#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:50,248 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/e565be15bcf349659099ec9d9a8756a6 is 50, key is test_row_0/B:col10/1732137288557/Put/seqid=0 2024-11-20T21:14:50,252 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:50,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742077_1253 (size=13425) 2024-11-20T21:14:50,265 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411203ff0bec83dbf4ca7a9a0540fb6ff2f3d_917d9f6cd1adb0d5f4ed70d067a87108 store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:50,285 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/e565be15bcf349659099ec9d9a8756a6 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/e565be15bcf349659099ec9d9a8756a6 2024-11-20T21:14:50,308 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411203ff0bec83dbf4ca7a9a0540fb6ff2f3d_917d9f6cd1adb0d5f4ed70d067a87108, store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:50,308 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203ff0bec83dbf4ca7a9a0540fb6ff2f3d_917d9f6cd1adb0d5f4ed70d067a87108 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:50,317 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 917d9f6cd1adb0d5f4ed70d067a87108/B of 917d9f6cd1adb0d5f4ed70d067a87108 into e565be15bcf349659099ec9d9a8756a6(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:50,317 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:50,317 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., storeName=917d9f6cd1adb0d5f4ed70d067a87108/B, priority=13, startTime=1732137290235; duration=0sec 2024-11-20T21:14:50,318 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:14:50,318 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:B 2024-11-20T21:14:50,318 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:14:50,329 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:14:50,329 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 917d9f6cd1adb0d5f4ed70d067a87108/C is initiating minor compaction (all files) 2024-11-20T21:14:50,329 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 917d9f6cd1adb0d5f4ed70d067a87108/C in TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:50,329 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/4001b324cf2548bc86415345e2346110, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/8a21a8e9974a4e2287c08530d3962808, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/0b707b541b0d4f1c9fc632911befb716] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp, totalSize=37.0 K 2024-11-20T21:14:50,332 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 4001b324cf2548bc86415345e2346110, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732137284801 2024-11-20T21:14:50,333 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a21a8e9974a4e2287c08530d3962808, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=455, earliestPutTs=1732137285129 2024-11-20T21:14:50,333 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b707b541b0d4f1c9fc632911befb716, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1732137286368 2024-11-20T21:14:50,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742078_1254 (size=4469) 2024-11-20T21:14:50,345 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 917d9f6cd1adb0d5f4ed70d067a87108#A#compaction#205 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:50,346 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/946431bd1f724eb5a813be421da50384 is 175, key is test_row_0/A:col10/1732137288557/Put/seqid=0 2024-11-20T21:14:50,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742079_1255 (size=32379) 2024-11-20T21:14:50,360 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 917d9f6cd1adb0d5f4ed70d067a87108#C#compaction#206 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:14:50,361 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/5d510ab53901435186d13ec5894708f1 is 50, key is test_row_0/C:col10/1732137288557/Put/seqid=0 2024-11-20T21:14:50,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742080_1256 (size=13425) 2024-11-20T21:14:50,373 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/5d510ab53901435186d13ec5894708f1 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/5d510ab53901435186d13ec5894708f1 2024-11-20T21:14:50,389 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 917d9f6cd1adb0d5f4ed70d067a87108/C of 917d9f6cd1adb0d5f4ed70d067a87108 into 5d510ab53901435186d13ec5894708f1(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:50,390 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:50,390 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., storeName=917d9f6cd1adb0d5f4ed70d067a87108/C, priority=13, startTime=1732137290235; duration=0sec 2024-11-20T21:14:50,390 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:50,390 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:C 2024-11-20T21:14:50,769 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/946431bd1f724eb5a813be421da50384 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/946431bd1f724eb5a813be421da50384 2024-11-20T21:14:50,776 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 917d9f6cd1adb0d5f4ed70d067a87108/A of 917d9f6cd1adb0d5f4ed70d067a87108 into 946431bd1f724eb5a813be421da50384(size=31.6 K), total size for store is 31.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:14:50,776 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 917d9f6cd1adb0d5f4ed70d067a87108: 2024-11-20T21:14:50,776 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., storeName=917d9f6cd1adb0d5f4ed70d067a87108/A, priority=13, startTime=1732137290235; duration=0sec 2024-11-20T21:14:50,776 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:14:50,777 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 917d9f6cd1adb0d5f4ed70d067a87108:A 2024-11-20T21:14:54,815 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T21:14:56,859 INFO [Thread-613 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-613. 2024-11-20T21:14:56,859 DEBUG [Thread-613 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:14:56,859 DEBUG [Thread-613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:56,859 DEBUG [Thread-613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:56,859 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:14:56,939 INFO [Thread-609 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-609. 2024-11-20T21:14:56,939 DEBUG [Thread-609 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:14:56,939 DEBUG [Thread-609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:56,939 DEBUG [Thread-609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:56,939 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:14:56,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T21:14:56,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-20T21:14:56,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 123 2024-11-20T21:14:56,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-20T21:14:56,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-20T21:14:56,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-20T21:14:56,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T21:14:56,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2000 2024-11-20T21:14:56,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1983 2024-11-20T21:14:56,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T21:14:56,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 928 2024-11-20T21:14:56,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2783 rows 2024-11-20T21:14:56,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 910 2024-11-20T21:14:56,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2730 rows 2024-11-20T21:14:56,940 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T21:14:56,940 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T21:14:56,940 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:56,940 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:14:56,940 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:14:56,941 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-20T21:14:56,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T21:14:56,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T21:14:56,944 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732137296944"}]},"ts":"1732137296944"} 2024-11-20T21:14:56,946 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T21:14:56,946 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T21:14:56,947 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T21:14:56,948 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=917d9f6cd1adb0d5f4ed70d067a87108, UNASSIGN}] 2024-11-20T21:14:56,949 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=917d9f6cd1adb0d5f4ed70d067a87108, UNASSIGN 2024-11-20T21:14:56,949 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=917d9f6cd1adb0d5f4ed70d067a87108, regionState=CLOSING, regionLocation=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:14:56,951 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=55, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=917d9f6cd1adb0d5f4ed70d067a87108, UNASSIGN because future has completed 2024-11-20T21:14:56,951 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T21:14:56,951 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536}] 2024-11-20T21:14:57,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T21:14:57,103 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(122): Close 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:57,103 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T21:14:57,103 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1722): Closing 917d9f6cd1adb0d5f4ed70d067a87108, disabling compactions & flushes 2024-11-20T21:14:57,103 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:57,103 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:57,103 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. after waiting 0 ms 2024-11-20T21:14:57,104 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:57,104 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(2902): Flushing 917d9f6cd1adb0d5f4ed70d067a87108 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-20T21:14:57,104 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=A 2024-11-20T21:14:57,104 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:57,104 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=B 2024-11-20T21:14:57,104 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:57,104 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 917d9f6cd1adb0d5f4ed70d067a87108, store=C 2024-11-20T21:14:57,104 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:14:57,111 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f7fc2c5968cc47d9bf5b1d88fcca9025_917d9f6cd1adb0d5f4ed70d067a87108 is 50, key is test_row_0/A:col10/1732137296937/Put/seqid=0 2024-11-20T21:14:57,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742081_1257 (size=9914) 2024-11-20T21:14:57,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T21:14:57,516 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:57,521 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f7fc2c5968cc47d9bf5b1d88fcca9025_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f7fc2c5968cc47d9bf5b1d88fcca9025_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:57,525 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/bca4d2ad6f994384956a2032453c134c, store: [table=TestAcidGuarantees family=A region=917d9f6cd1adb0d5f4ed70d067a87108] 2024-11-20T21:14:57,526 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/bca4d2ad6f994384956a2032453c134c is 175, key is test_row_0/A:col10/1732137296937/Put/seqid=0 2024-11-20T21:14:57,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742082_1258 (size=22561) 2024-11-20T21:14:57,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T21:14:57,932 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=484, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/bca4d2ad6f994384956a2032453c134c 2024-11-20T21:14:57,939 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/24f0b17a3c424161959baaa90652fb9c is 50, key is test_row_0/B:col10/1732137296937/Put/seqid=0 2024-11-20T21:14:57,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742083_1259 (size=9857) 2024-11-20T21:14:58,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T21:14:58,343 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/24f0b17a3c424161959baaa90652fb9c 2024-11-20T21:14:58,351 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/1186ecb341dd4c85ba48ec0922137615 is 50, key is test_row_0/C:col10/1732137296937/Put/seqid=0 2024-11-20T21:14:58,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742084_1260 (size=9857) 2024-11-20T21:14:58,756 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=484 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/1186ecb341dd4c85ba48ec0922137615 2024-11-20T21:14:58,761 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/A/bca4d2ad6f994384956a2032453c134c as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/bca4d2ad6f994384956a2032453c134c 2024-11-20T21:14:58,764 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/bca4d2ad6f994384956a2032453c134c, entries=100, sequenceid=484, filesize=22.0 K 2024-11-20T21:14:58,765 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/B/24f0b17a3c424161959baaa90652fb9c as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/24f0b17a3c424161959baaa90652fb9c 2024-11-20T21:14:58,768 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/24f0b17a3c424161959baaa90652fb9c, entries=100, sequenceid=484, filesize=9.6 K 2024-11-20T21:14:58,768 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/.tmp/C/1186ecb341dd4c85ba48ec0922137615 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/1186ecb341dd4c85ba48ec0922137615 2024-11-20T21:14:58,771 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/1186ecb341dd4c85ba48ec0922137615, entries=100, sequenceid=484, filesize=9.6 K 2024-11-20T21:14:58,772 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 917d9f6cd1adb0d5f4ed70d067a87108 in 1668ms, sequenceid=484, compaction requested=false 2024-11-20T21:14:58,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/4c7ae6631a9f46c7b1c506f50f359276, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/0b9ca9e5774a4faba145c0c9fa598068, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/302735d4e24c4ffaa00f5bcd99557e8b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/f60d019128074d2a9efd4b3c22413da0, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/036c165b18f64c5fb1bd931fd43c5860, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/59951e166bff4857ab64443c3126d9dd, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/ed2f9165bed84c5b8b4ba260d50ffbcc, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/4a4a90528ad34edb86eb915da6cd6de3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/833ed27cc00047bca9055df34493ec1c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/159e3e7b47a44f5f8d9c94f8b653eb9f, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/e92ba6f1785d4468b3d82c31fa6e1e84, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/cb9950d0c6a946a48f13d8d6fda639c7, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/10d4be816bda4cf081e9c90c40228296, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/a40ffb66b0004ec28f4185f8ac836f84, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/3e2030e1ed2b4b9da565c417ea3458f3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/8f5303fa4c964d47ad1a99fcedd341b9, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/c3a2027d114c4d179d794c21fab5c5fc, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/b644816e0e084f4384c233342c1ac260, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/16007388d96047288b7787e6ccc39c7e, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/b616d5ba34884772b73bf200e0f5d27e, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/4afcf0c6dcbb4325b4fba11b94ed2590, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/14241ce8fdba4fdc914436adacc5897c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/8a07dd0dce914027aeebb912e565360f, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/2d25d282b9454559be977af979198b49, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/60894b0cf0c84cbdafcf26c5bb2bedcd, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/eb7cd2a4292c489f8e5b96d11042e7b1, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/8258ceac06ea4e0da9753744956e5d41, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/3477dc4643ca41298183fa8f367b9be2, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/0e5c8a518a784128ba4a8c1383b347b3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/130daac1a3bf4ff4b0086ebce89e639f, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/b622be437475486cb8f295be57d1c19f, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/75c27e017b9f429c8b98b5e51ac91f64, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/9932934ba0034ec3a1caa7c1e4c71b7f] to archive 2024-11-20T21:14:58,774 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T21:14:58,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/4c7ae6631a9f46c7b1c506f50f359276 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/4c7ae6631a9f46c7b1c506f50f359276 2024-11-20T21:14:58,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/0b9ca9e5774a4faba145c0c9fa598068 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/0b9ca9e5774a4faba145c0c9fa598068 2024-11-20T21:14:58,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/302735d4e24c4ffaa00f5bcd99557e8b to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/302735d4e24c4ffaa00f5bcd99557e8b 2024-11-20T21:14:58,778 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/f60d019128074d2a9efd4b3c22413da0 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/f60d019128074d2a9efd4b3c22413da0 2024-11-20T21:14:58,779 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/036c165b18f64c5fb1bd931fd43c5860 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/036c165b18f64c5fb1bd931fd43c5860 2024-11-20T21:14:58,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/59951e166bff4857ab64443c3126d9dd to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/59951e166bff4857ab64443c3126d9dd 2024-11-20T21:14:58,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/ed2f9165bed84c5b8b4ba260d50ffbcc to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/ed2f9165bed84c5b8b4ba260d50ffbcc 2024-11-20T21:14:58,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/4a4a90528ad34edb86eb915da6cd6de3 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/4a4a90528ad34edb86eb915da6cd6de3 2024-11-20T21:14:58,782 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/833ed27cc00047bca9055df34493ec1c to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/833ed27cc00047bca9055df34493ec1c 2024-11-20T21:14:58,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/159e3e7b47a44f5f8d9c94f8b653eb9f to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/159e3e7b47a44f5f8d9c94f8b653eb9f 2024-11-20T21:14:58,785 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/e92ba6f1785d4468b3d82c31fa6e1e84 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/e92ba6f1785d4468b3d82c31fa6e1e84 2024-11-20T21:14:58,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/cb9950d0c6a946a48f13d8d6fda639c7 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/cb9950d0c6a946a48f13d8d6fda639c7 2024-11-20T21:14:58,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/10d4be816bda4cf081e9c90c40228296 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/10d4be816bda4cf081e9c90c40228296 2024-11-20T21:14:58,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/a40ffb66b0004ec28f4185f8ac836f84 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/a40ffb66b0004ec28f4185f8ac836f84 2024-11-20T21:14:58,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/3e2030e1ed2b4b9da565c417ea3458f3 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/3e2030e1ed2b4b9da565c417ea3458f3 2024-11-20T21:14:58,790 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/8f5303fa4c964d47ad1a99fcedd341b9 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/8f5303fa4c964d47ad1a99fcedd341b9 2024-11-20T21:14:58,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/c3a2027d114c4d179d794c21fab5c5fc to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/c3a2027d114c4d179d794c21fab5c5fc 2024-11-20T21:14:58,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/b644816e0e084f4384c233342c1ac260 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/b644816e0e084f4384c233342c1ac260 2024-11-20T21:14:58,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/16007388d96047288b7787e6ccc39c7e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/16007388d96047288b7787e6ccc39c7e 2024-11-20T21:14:58,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/b616d5ba34884772b73bf200e0f5d27e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/b616d5ba34884772b73bf200e0f5d27e 2024-11-20T21:14:58,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/4afcf0c6dcbb4325b4fba11b94ed2590 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/4afcf0c6dcbb4325b4fba11b94ed2590 2024-11-20T21:14:58,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/14241ce8fdba4fdc914436adacc5897c to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/14241ce8fdba4fdc914436adacc5897c 2024-11-20T21:14:58,797 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/8a07dd0dce914027aeebb912e565360f to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/8a07dd0dce914027aeebb912e565360f 2024-11-20T21:14:58,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/2d25d282b9454559be977af979198b49 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/2d25d282b9454559be977af979198b49 2024-11-20T21:14:58,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/60894b0cf0c84cbdafcf26c5bb2bedcd to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/60894b0cf0c84cbdafcf26c5bb2bedcd 2024-11-20T21:14:58,800 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/eb7cd2a4292c489f8e5b96d11042e7b1 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/eb7cd2a4292c489f8e5b96d11042e7b1 2024-11-20T21:14:58,801 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/8258ceac06ea4e0da9753744956e5d41 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/8258ceac06ea4e0da9753744956e5d41 2024-11-20T21:14:58,802 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/3477dc4643ca41298183fa8f367b9be2 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/3477dc4643ca41298183fa8f367b9be2 2024-11-20T21:14:58,803 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/0e5c8a518a784128ba4a8c1383b347b3 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/0e5c8a518a784128ba4a8c1383b347b3 2024-11-20T21:14:58,804 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/130daac1a3bf4ff4b0086ebce89e639f to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/130daac1a3bf4ff4b0086ebce89e639f 2024-11-20T21:14:58,805 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/b622be437475486cb8f295be57d1c19f to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/b622be437475486cb8f295be57d1c19f 2024-11-20T21:14:58,806 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/75c27e017b9f429c8b98b5e51ac91f64 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/75c27e017b9f429c8b98b5e51ac91f64 2024-11-20T21:14:58,807 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/9932934ba0034ec3a1caa7c1e4c71b7f to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/9932934ba0034ec3a1caa7c1e4c71b7f 2024-11-20T21:14:58,808 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/3eb734e966674494902636e819cd6db9, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/bf8fb2f64c744f2d8a2f71559ec37d64, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/d3085ff7756847a9b4faaa620ce3725c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/887fcf60681e4fb78cd9cf4dc9aafb58, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/5006763660844be9be657840edb975dc, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/3bb27a91313842f6808c63336c55a976, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/d383cb01ce774d8c96e97a32f6369621, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/238cd38c7a4f4def86e2685b8503be52, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/4e79197c432849a8a3053073942375ac, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/8a8fbed3603044a4839147270e16621c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/e8f6a45bbb4345a185c6c938c2ef46dd, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/5fc403c4aa3c4a3590e21b84e18b8f92, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/174056d93c1c46d08393d828357eb0c1, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/28ce742cee1d4b85be7a0b33cef03a82, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/4bdd7a11231d4931919f6861fc9e2df5, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/8dc85ee02f954cd5af0d695785e22534, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/0fade51b0a324076bac7484012c068d5, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/875c2ab7b2a9487188e31614c0b22751, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/1c8c41e0fd4446b999bb1b84baa94621, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/4c880ee597ab4d66b1c17915ecd32d4d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/ffe9bc2c79484fdd8b3fa46b14a5adc7, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/2525a0a6c7d2421ebf396baf6c82ecf8, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/f987ce5be4274b21ad06f56b6a4609db, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/d5a21c7a6b9b4b579697ac8f17744655, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/8904667fa50e442eb19e1378d0d78fdb, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/cbc830a87d9c46b7bfeba96dcaab7f3c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/1ddaf799408147dca6014f71305ab33c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/9ca5a944b32b4c8cbc6a3e6e236ce4fa, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/fd50d4388ef9414f832fd4ed83a8e9e5, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/3ca443a1b1044673ab91a81423e4456b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/a72012745fa7422a9ac29a037a5ed076, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/49df26a70bd04f4eab43214fd3ff634b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/abcd3679d84e4d60882bcbfd0bbc41e2] to archive 2024-11-20T21:14:58,809 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T21:14:58,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/3eb734e966674494902636e819cd6db9 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/3eb734e966674494902636e819cd6db9 2024-11-20T21:14:58,812 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/bf8fb2f64c744f2d8a2f71559ec37d64 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/bf8fb2f64c744f2d8a2f71559ec37d64 2024-11-20T21:14:58,813 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/d3085ff7756847a9b4faaa620ce3725c to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/d3085ff7756847a9b4faaa620ce3725c 2024-11-20T21:14:58,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/887fcf60681e4fb78cd9cf4dc9aafb58 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/887fcf60681e4fb78cd9cf4dc9aafb58 2024-11-20T21:14:58,815 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/5006763660844be9be657840edb975dc to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/5006763660844be9be657840edb975dc 2024-11-20T21:14:58,816 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/3bb27a91313842f6808c63336c55a976 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/3bb27a91313842f6808c63336c55a976 2024-11-20T21:14:58,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/d383cb01ce774d8c96e97a32f6369621 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/d383cb01ce774d8c96e97a32f6369621 2024-11-20T21:14:58,818 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/238cd38c7a4f4def86e2685b8503be52 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/238cd38c7a4f4def86e2685b8503be52 2024-11-20T21:14:58,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/4e79197c432849a8a3053073942375ac to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/4e79197c432849a8a3053073942375ac 2024-11-20T21:14:58,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/8a8fbed3603044a4839147270e16621c to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/8a8fbed3603044a4839147270e16621c 2024-11-20T21:14:58,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/e8f6a45bbb4345a185c6c938c2ef46dd to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/e8f6a45bbb4345a185c6c938c2ef46dd 2024-11-20T21:14:58,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/5fc403c4aa3c4a3590e21b84e18b8f92 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/5fc403c4aa3c4a3590e21b84e18b8f92 2024-11-20T21:14:58,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/174056d93c1c46d08393d828357eb0c1 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/174056d93c1c46d08393d828357eb0c1 2024-11-20T21:14:58,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/28ce742cee1d4b85be7a0b33cef03a82 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/28ce742cee1d4b85be7a0b33cef03a82 2024-11-20T21:14:58,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/4bdd7a11231d4931919f6861fc9e2df5 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/4bdd7a11231d4931919f6861fc9e2df5 2024-11-20T21:14:58,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/8dc85ee02f954cd5af0d695785e22534 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/8dc85ee02f954cd5af0d695785e22534 2024-11-20T21:14:58,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/0fade51b0a324076bac7484012c068d5 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/0fade51b0a324076bac7484012c068d5 2024-11-20T21:14:58,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/875c2ab7b2a9487188e31614c0b22751 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/875c2ab7b2a9487188e31614c0b22751 2024-11-20T21:14:58,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/1c8c41e0fd4446b999bb1b84baa94621 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/1c8c41e0fd4446b999bb1b84baa94621 2024-11-20T21:14:58,831 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/4c880ee597ab4d66b1c17915ecd32d4d to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/4c880ee597ab4d66b1c17915ecd32d4d 2024-11-20T21:14:58,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/ffe9bc2c79484fdd8b3fa46b14a5adc7 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/ffe9bc2c79484fdd8b3fa46b14a5adc7 2024-11-20T21:14:58,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/2525a0a6c7d2421ebf396baf6c82ecf8 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/2525a0a6c7d2421ebf396baf6c82ecf8 2024-11-20T21:14:58,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/f987ce5be4274b21ad06f56b6a4609db to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/f987ce5be4274b21ad06f56b6a4609db 2024-11-20T21:14:58,835 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/d5a21c7a6b9b4b579697ac8f17744655 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/d5a21c7a6b9b4b579697ac8f17744655 2024-11-20T21:14:58,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/8904667fa50e442eb19e1378d0d78fdb to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/8904667fa50e442eb19e1378d0d78fdb 2024-11-20T21:14:58,839 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/cbc830a87d9c46b7bfeba96dcaab7f3c to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/cbc830a87d9c46b7bfeba96dcaab7f3c 2024-11-20T21:14:58,840 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/1ddaf799408147dca6014f71305ab33c to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/1ddaf799408147dca6014f71305ab33c 2024-11-20T21:14:58,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/9ca5a944b32b4c8cbc6a3e6e236ce4fa to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/9ca5a944b32b4c8cbc6a3e6e236ce4fa 2024-11-20T21:14:58,842 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/fd50d4388ef9414f832fd4ed83a8e9e5 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/fd50d4388ef9414f832fd4ed83a8e9e5 2024-11-20T21:14:58,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/3ca443a1b1044673ab91a81423e4456b to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/3ca443a1b1044673ab91a81423e4456b 2024-11-20T21:14:58,844 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/a72012745fa7422a9ac29a037a5ed076 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/a72012745fa7422a9ac29a037a5ed076 2024-11-20T21:14:58,845 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/49df26a70bd04f4eab43214fd3ff634b to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/49df26a70bd04f4eab43214fd3ff634b 2024-11-20T21:14:58,846 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/abcd3679d84e4d60882bcbfd0bbc41e2 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/abcd3679d84e4d60882bcbfd0bbc41e2 2024-11-20T21:14:58,848 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/f002de1bec3343b2830bb54e392cbb21, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/4cf4f1e16383444dbeeeee6d728e0e74, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/3e517de421c64dd8a7f06e5ab417af16, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/78af7d6edcc24fbd8503964a541cddfc, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/fce52246e3164f9d8c6b7aa16edb5861, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/54f37ed9bafb47e4be65153c408e371f, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/179fecd5651747a1acd6fddb0f6d24ac, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/cc4f7501892841ca95fa2b8b6b4e94cd, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/e4cafc8b620b4e1a958b1bf8086b5776, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/ac35c7516a654afcbb0825cc745e2bfb, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/2b4c5d3f33574a10919f5b770554d659, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/393a26f23d1345978e4a89e16780bb46, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/912795a83ec64752a89bdcc80c4b1518, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/31f0bf8b7fb248b698ca6278d59eb921, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/8a2e960de3464409b1430bb4b5735428, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/882d6fe22a944cfea0118c8f0bad6398, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/1c4bfb3695034a848a92a6866a67b3d3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/85d8701f7bf24b9ab6c4f8de75cef0d2, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/1e42c9dc636248398ef07fecf42a9348, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/16bd3f9e593b4ca5b289af9262533b99, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/c2a6b958059d47ab8bd2f79462a23f82, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/e2d402c037154bb69d02a86a2790e5cb, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/86ebc7091762433d9e103edb2a8f99f1, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/9b8de26d699f4e1d953eee3dc9b7f970, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/d968c15f9b674847ab50477d79ae2cb4, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/be91b99c028548849637509bddbbfdab, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/8cedc98d4c9248dbb1d4bcfbc43078f5, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/59c4e56acc88417ba933f68621a184b3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/17cd706b17e34452af12fe92944b8c5e, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/4001b324cf2548bc86415345e2346110, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/35f09cd1db4e45c3ad096a338a87e46f, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/8a21a8e9974a4e2287c08530d3962808, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/0b707b541b0d4f1c9fc632911befb716] to archive 2024-11-20T21:14:58,849 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T21:14:58,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/f002de1bec3343b2830bb54e392cbb21 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/f002de1bec3343b2830bb54e392cbb21 2024-11-20T21:14:58,851 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/4cf4f1e16383444dbeeeee6d728e0e74 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/4cf4f1e16383444dbeeeee6d728e0e74 2024-11-20T21:14:58,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/3e517de421c64dd8a7f06e5ab417af16 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/3e517de421c64dd8a7f06e5ab417af16 2024-11-20T21:14:58,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/78af7d6edcc24fbd8503964a541cddfc to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/78af7d6edcc24fbd8503964a541cddfc 2024-11-20T21:14:58,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/fce52246e3164f9d8c6b7aa16edb5861 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/fce52246e3164f9d8c6b7aa16edb5861 2024-11-20T21:14:58,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/54f37ed9bafb47e4be65153c408e371f to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/54f37ed9bafb47e4be65153c408e371f 2024-11-20T21:14:58,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/179fecd5651747a1acd6fddb0f6d24ac to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/179fecd5651747a1acd6fddb0f6d24ac 2024-11-20T21:14:58,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/cc4f7501892841ca95fa2b8b6b4e94cd to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/cc4f7501892841ca95fa2b8b6b4e94cd 2024-11-20T21:14:58,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/e4cafc8b620b4e1a958b1bf8086b5776 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/e4cafc8b620b4e1a958b1bf8086b5776 2024-11-20T21:14:58,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/ac35c7516a654afcbb0825cc745e2bfb to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/ac35c7516a654afcbb0825cc745e2bfb 2024-11-20T21:14:58,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/2b4c5d3f33574a10919f5b770554d659 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/2b4c5d3f33574a10919f5b770554d659 2024-11-20T21:14:58,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/393a26f23d1345978e4a89e16780bb46 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/393a26f23d1345978e4a89e16780bb46 2024-11-20T21:14:58,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/912795a83ec64752a89bdcc80c4b1518 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/912795a83ec64752a89bdcc80c4b1518 2024-11-20T21:14:58,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/31f0bf8b7fb248b698ca6278d59eb921 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/31f0bf8b7fb248b698ca6278d59eb921 2024-11-20T21:14:58,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/8a2e960de3464409b1430bb4b5735428 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/8a2e960de3464409b1430bb4b5735428 2024-11-20T21:14:58,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/882d6fe22a944cfea0118c8f0bad6398 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/882d6fe22a944cfea0118c8f0bad6398 2024-11-20T21:14:58,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/1c4bfb3695034a848a92a6866a67b3d3 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/1c4bfb3695034a848a92a6866a67b3d3 2024-11-20T21:14:58,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/85d8701f7bf24b9ab6c4f8de75cef0d2 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/85d8701f7bf24b9ab6c4f8de75cef0d2 2024-11-20T21:14:58,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/1e42c9dc636248398ef07fecf42a9348 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/1e42c9dc636248398ef07fecf42a9348 2024-11-20T21:14:58,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/16bd3f9e593b4ca5b289af9262533b99 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/16bd3f9e593b4ca5b289af9262533b99 2024-11-20T21:14:58,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/c2a6b958059d47ab8bd2f79462a23f82 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/c2a6b958059d47ab8bd2f79462a23f82 2024-11-20T21:14:58,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/e2d402c037154bb69d02a86a2790e5cb to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/e2d402c037154bb69d02a86a2790e5cb 2024-11-20T21:14:58,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/86ebc7091762433d9e103edb2a8f99f1 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/86ebc7091762433d9e103edb2a8f99f1 2024-11-20T21:14:58,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/9b8de26d699f4e1d953eee3dc9b7f970 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/9b8de26d699f4e1d953eee3dc9b7f970 2024-11-20T21:14:58,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/d968c15f9b674847ab50477d79ae2cb4 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/d968c15f9b674847ab50477d79ae2cb4 2024-11-20T21:14:58,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/be91b99c028548849637509bddbbfdab to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/be91b99c028548849637509bddbbfdab 2024-11-20T21:14:58,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/8cedc98d4c9248dbb1d4bcfbc43078f5 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/8cedc98d4c9248dbb1d4bcfbc43078f5 2024-11-20T21:14:58,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/59c4e56acc88417ba933f68621a184b3 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/59c4e56acc88417ba933f68621a184b3 2024-11-20T21:14:58,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/17cd706b17e34452af12fe92944b8c5e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/17cd706b17e34452af12fe92944b8c5e 2024-11-20T21:14:58,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/4001b324cf2548bc86415345e2346110 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/4001b324cf2548bc86415345e2346110 2024-11-20T21:14:58,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/35f09cd1db4e45c3ad096a338a87e46f to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/35f09cd1db4e45c3ad096a338a87e46f 2024-11-20T21:14:58,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/8a21a8e9974a4e2287c08530d3962808 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/8a21a8e9974a4e2287c08530d3962808 2024-11-20T21:14:58,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/0b707b541b0d4f1c9fc632911befb716 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/0b707b541b0d4f1c9fc632911befb716 2024-11-20T21:14:58,897 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/recovered.edits/487.seqid, newMaxSeqId=487, maxSeqId=4 2024-11-20T21:14:58,898 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. 2024-11-20T21:14:58,898 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] regionserver.HRegion(1676): Region close journal for 917d9f6cd1adb0d5f4ed70d067a87108: Waiting for close lock at 1732137297103Running coprocessor pre-close hooks at 1732137297103Disabling compacts and flushes for region at 1732137297103Disabling writes for close at 1732137297103Obtaining lock to block concurrent updates at 1732137297104 (+1 ms)Preparing flush snapshotting stores in 917d9f6cd1adb0d5f4ed70d067a87108 at 1732137297104Finished memstore snapshotting TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732137297104Flushing stores of TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108. at 1732137297105 (+1 ms)Flushing 917d9f6cd1adb0d5f4ed70d067a87108/A: creating writer at 1732137297105Flushing mob file 917d9f6cd1adb0d5f4ed70d067a87108/A: appending metadata at 1732137297111 (+6 ms)Flushing mob file 917d9f6cd1adb0d5f4ed70d067a87108/A: closing flushed file at 1732137297111Flushing 917d9f6cd1adb0d5f4ed70d067a87108/A: appending metadata at 1732137297525 (+414 ms)Flushing 917d9f6cd1adb0d5f4ed70d067a87108/A: closing flushed file at 1732137297525Flushing 917d9f6cd1adb0d5f4ed70d067a87108/B: creating writer at 1732137297936 (+411 ms)Flushing 917d9f6cd1adb0d5f4ed70d067a87108/B: appending metadata at 1732137297938 (+2 ms)Flushing 917d9f6cd1adb0d5f4ed70d067a87108/B: closing flushed file at 1732137297938Flushing 917d9f6cd1adb0d5f4ed70d067a87108/C: creating writer at 1732137298348 (+410 ms)Flushing 917d9f6cd1adb0d5f4ed70d067a87108/C: appending metadata at 1732137298351 (+3 ms)Flushing 917d9f6cd1adb0d5f4ed70d067a87108/C: closing flushed file at 1732137298351Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18cbbfa7: reopening flushed file at 1732137298760 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1261833e: reopening flushed file at 1732137298764 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2fa92dee: reopening flushed file at 1732137298768 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 917d9f6cd1adb0d5f4ed70d067a87108 in 1668ms, sequenceid=484, compaction requested=false at 1732137298772 (+4 ms)Writing region close event to WAL at 1732137298892 (+120 ms)Running coprocessor post-close hooks at 1732137298898 (+6 ms)Closed at 1732137298898 2024-11-20T21:14:58,901 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=57}] handler.UnassignRegionHandler(157): Closed 917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:58,901 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=56 updating hbase:meta row=917d9f6cd1adb0d5f4ed70d067a87108, regionState=CLOSED 2024-11-20T21:14:58,903 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, ppid=56, state=RUNNABLE, hasLock=false; CloseRegionProcedure 917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 because future has completed 2024-11-20T21:14:58,907 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-20T21:14:58,907 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseRegionProcedure 917d9f6cd1adb0d5f4ed70d067a87108, server=0b16ab3f13b7,36987,1732137236536 in 1.9540 sec 2024-11-20T21:14:58,909 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=55 2024-11-20T21:14:58,909 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=55, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=917d9f6cd1adb0d5f4ed70d067a87108, UNASSIGN in 1.9590 sec 2024-11-20T21:14:58,911 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-20T21:14:58,911 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9630 sec 2024-11-20T21:14:58,913 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732137298913"}]},"ts":"1732137298913"} 2024-11-20T21:14:58,914 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T21:14:58,914 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T21:14:58,915 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9730 sec 2024-11-20T21:14:59,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-20T21:14:59,087 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:14:59,087 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-20T21:14:59,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:14:59,089 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=58, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:14:59,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-20T21:14:59,090 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=58, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:14:59,093 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:59,095 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A, FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B, FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C, FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/recovered.edits] 2024-11-20T21:14:59,098 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/946431bd1f724eb5a813be421da50384 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/946431bd1f724eb5a813be421da50384 2024-11-20T21:14:59,099 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/bca4d2ad6f994384956a2032453c134c to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/A/bca4d2ad6f994384956a2032453c134c 2024-11-20T21:14:59,101 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/24f0b17a3c424161959baaa90652fb9c to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/24f0b17a3c424161959baaa90652fb9c 2024-11-20T21:14:59,102 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/e565be15bcf349659099ec9d9a8756a6 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/B/e565be15bcf349659099ec9d9a8756a6 2024-11-20T21:14:59,104 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/1186ecb341dd4c85ba48ec0922137615 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/1186ecb341dd4c85ba48ec0922137615 2024-11-20T21:14:59,105 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/5d510ab53901435186d13ec5894708f1 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/C/5d510ab53901435186d13ec5894708f1 2024-11-20T21:14:59,110 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/recovered.edits/487.seqid to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108/recovered.edits/487.seqid 2024-11-20T21:14:59,111 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:59,111 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T21:14:59,112 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T21:14:59,113 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-20T21:14:59,117 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200ddc92c7c3054e59859ef161d98dc46b_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200ddc92c7c3054e59859ef161d98dc46b_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:59,118 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201887a5641bc64b518f5455a9120a43cf_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201887a5641bc64b518f5455a9120a43cf_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:59,119 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202947acca4c19456097a6b64bad8f28ec_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202947acca4c19456097a6b64bad8f28ec_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:59,120 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202db99cc3d72d4e6fb29ab9dab163fd60_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202db99cc3d72d4e6fb29ab9dab163fd60_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:59,121 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411204654345419db4fd3b2625ea95381bae0_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411204654345419db4fd3b2625ea95381bae0_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:59,122 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411204a7b1c0394e44f92b6a58efc163b7571_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411204a7b1c0394e44f92b6a58efc163b7571_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:59,123 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411204e355f4eda4342e1b8a09ed5a44c7c77_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411204e355f4eda4342e1b8a09ed5a44c7c77_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:59,124 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120626e32ff13d343f0ac6287cc0e27f5ef_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120626e32ff13d343f0ac6287cc0e27f5ef_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:59,125 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120800a69997f524edbb13e7b8b4cc5f456_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120800a69997f524edbb13e7b8b4cc5f456_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:59,126 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112089404cbe7c6f4d4bba6c6c2c7c314ebd_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112089404cbe7c6f4d4bba6c6c2c7c314ebd_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:59,127 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208db8dbebad19491f9d6688ada8e64248_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208db8dbebad19491f9d6688ada8e64248_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:59,129 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209a6663ad9bc64646ab5bf88bfee65282_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209a6663ad9bc64646ab5bf88bfee65282_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:59,130 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209ad8eddf2f5e4dca8de8531396273f4c_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209ad8eddf2f5e4dca8de8531396273f4c_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:59,130 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120aae157af444c46fe9d71e56ada58a5fe_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120aae157af444c46fe9d71e56ada58a5fe_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:59,131 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120aba22873a9cc4a79b64f83a99aa8b4f5_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120aba22873a9cc4a79b64f83a99aa8b4f5_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:59,132 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120aee4a0a231ee481baadccca4f3a3ffea_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120aee4a0a231ee481baadccca4f3a3ffea_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:59,133 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c6fce599a1e44b6b96ebfdd2b720edc9_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c6fce599a1e44b6b96ebfdd2b720edc9_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:59,135 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c8515c10512b4890a1c494e9d702092d_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c8515c10512b4890a1c494e9d702092d_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:59,136 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ca6a534f13d943b7838e7d0754804a69_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ca6a534f13d943b7838e7d0754804a69_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:59,137 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120cb961bdddf01402b87da7b72642ae17b_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120cb961bdddf01402b87da7b72642ae17b_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:59,138 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120cbba340b1c424f7286f228178faa4218_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120cbba340b1c424f7286f228178faa4218_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:59,139 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120dfa213983bd748ab8984d9af14e4f877_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120dfa213983bd748ab8984d9af14e4f877_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:59,140 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ea821fcf661d4ccc97bef84f62580464_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120ea821fcf661d4ccc97bef84f62580464_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:59,141 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f7fc2c5968cc47d9bf5b1d88fcca9025_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f7fc2c5968cc47d9bf5b1d88fcca9025_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:59,142 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f88e86d74d764ff8a86c1dc2dfd1a99b_917d9f6cd1adb0d5f4ed70d067a87108 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f88e86d74d764ff8a86c1dc2dfd1a99b_917d9f6cd1adb0d5f4ed70d067a87108 2024-11-20T21:14:59,143 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T21:14:59,145 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=58, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:14:59,147 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T21:14:59,149 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T21:14:59,150 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=58, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:14:59,150 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T21:14:59,150 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732137299150"}]},"ts":"9223372036854775807"} 2024-11-20T21:14:59,152 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T21:14:59,152 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 917d9f6cd1adb0d5f4ed70d067a87108, NAME => 'TestAcidGuarantees,,1732137264641.917d9f6cd1adb0d5f4ed70d067a87108.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T21:14:59,152 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T21:14:59,152 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732137299152"}]},"ts":"9223372036854775807"} 2024-11-20T21:14:59,153 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T21:14:59,154 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=58, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:14:59,155 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 66 msec 2024-11-20T21:14:59,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-20T21:14:59,197 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T21:14:59,197 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:14:59,212 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247 (was 247), OpenFileDescriptor=453 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=687 (was 567) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1574 (was 849) - AvailableMemoryMB LEAK? - 2024-11-20T21:14:59,227 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=247, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=687, ProcessCount=11, AvailableMemoryMB=1570 2024-11-20T21:14:59,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T21:14:59,229 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T21:14:59,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T21:14:59,232 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T21:14:59,232 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:14:59,232 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 59 2024-11-20T21:14:59,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-20T21:14:59,233 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T21:14:59,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742085_1261 (size=1061) 2024-11-20T21:14:59,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-20T21:14:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-20T21:14:59,647 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094 2024-11-20T21:14:59,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742086_1262 (size=53) 2024-11-20T21:14:59,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-20T21:15:00,053 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:15:00,053 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a5436f89d60238ad7387432d34167717, disabling compactions & flushes 2024-11-20T21:15:00,053 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:00,053 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:00,053 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. after waiting 0 ms 2024-11-20T21:15:00,053 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:00,053 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:00,053 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a5436f89d60238ad7387432d34167717: Waiting for close lock at 1732137300053Disabling compacts and flushes for region at 1732137300053Disabling writes for close at 1732137300053Writing region close event to WAL at 1732137300053Closed at 1732137300053 2024-11-20T21:15:00,055 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T21:15:00,055 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732137300055"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732137300055"}]},"ts":"1732137300055"} 2024-11-20T21:15:00,057 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T21:15:00,058 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T21:15:00,058 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732137300058"}]},"ts":"1732137300058"} 2024-11-20T21:15:00,072 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T21:15:00,073 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5436f89d60238ad7387432d34167717, ASSIGN}] 2024-11-20T21:15:00,074 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5436f89d60238ad7387432d34167717, ASSIGN 2024-11-20T21:15:00,076 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5436f89d60238ad7387432d34167717, ASSIGN; state=OFFLINE, location=0b16ab3f13b7,36987,1732137236536; forceNewPlan=false, retain=false 2024-11-20T21:15:00,227 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=a5436f89d60238ad7387432d34167717, regionState=OPENING, regionLocation=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:00,233 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5436f89d60238ad7387432d34167717, ASSIGN because future has completed 2024-11-20T21:15:00,237 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536}] 2024-11-20T21:15:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-20T21:15:00,395 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:00,395 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:15:00,395 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a5436f89d60238ad7387432d34167717 2024-11-20T21:15:00,396 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:15:00,396 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7794): checking encryption for a5436f89d60238ad7387432d34167717 2024-11-20T21:15:00,396 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(7797): checking classloading for a5436f89d60238ad7387432d34167717 2024-11-20T21:15:00,397 INFO [StoreOpener-a5436f89d60238ad7387432d34167717-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a5436f89d60238ad7387432d34167717 2024-11-20T21:15:00,399 INFO [StoreOpener-a5436f89d60238ad7387432d34167717-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T21:15:00,399 INFO [StoreOpener-a5436f89d60238ad7387432d34167717-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a5436f89d60238ad7387432d34167717 columnFamilyName A 2024-11-20T21:15:00,399 DEBUG [StoreOpener-a5436f89d60238ad7387432d34167717-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:00,399 INFO [StoreOpener-a5436f89d60238ad7387432d34167717-1 {}] regionserver.HStore(327): Store=a5436f89d60238ad7387432d34167717/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:15:00,400 INFO [StoreOpener-a5436f89d60238ad7387432d34167717-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a5436f89d60238ad7387432d34167717 2024-11-20T21:15:00,401 INFO [StoreOpener-a5436f89d60238ad7387432d34167717-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T21:15:00,401 INFO [StoreOpener-a5436f89d60238ad7387432d34167717-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a5436f89d60238ad7387432d34167717 columnFamilyName B 2024-11-20T21:15:00,401 DEBUG [StoreOpener-a5436f89d60238ad7387432d34167717-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:00,402 INFO [StoreOpener-a5436f89d60238ad7387432d34167717-1 {}] regionserver.HStore(327): Store=a5436f89d60238ad7387432d34167717/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:15:00,402 INFO [StoreOpener-a5436f89d60238ad7387432d34167717-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a5436f89d60238ad7387432d34167717 2024-11-20T21:15:00,403 INFO [StoreOpener-a5436f89d60238ad7387432d34167717-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T21:15:00,403 INFO [StoreOpener-a5436f89d60238ad7387432d34167717-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a5436f89d60238ad7387432d34167717 columnFamilyName C 2024-11-20T21:15:00,403 DEBUG [StoreOpener-a5436f89d60238ad7387432d34167717-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:00,403 INFO [StoreOpener-a5436f89d60238ad7387432d34167717-1 {}] regionserver.HStore(327): Store=a5436f89d60238ad7387432d34167717/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:15:00,404 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:00,404 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1038): replaying wal for a5436f89d60238ad7387432d34167717 2024-11-20T21:15:00,404 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717 2024-11-20T21:15:00,405 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717 2024-11-20T21:15:00,406 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1048): stopping wal replay for a5436f89d60238ad7387432d34167717 2024-11-20T21:15:00,406 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for a5436f89d60238ad7387432d34167717 2024-11-20T21:15:00,406 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T21:15:00,407 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1093): writing seq id for a5436f89d60238ad7387432d34167717 2024-11-20T21:15:00,409 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T21:15:00,410 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1114): Opened a5436f89d60238ad7387432d34167717; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71872100, jitterRate=0.07097774744033813}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T21:15:00,410 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a5436f89d60238ad7387432d34167717 2024-11-20T21:15:00,411 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegion(1006): Region open journal for a5436f89d60238ad7387432d34167717: Running coprocessor pre-open hook at 1732137300396Writing region info on filesystem at 1732137300396Initializing all the Stores at 1732137300397 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732137300397Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732137300397Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732137300397Cleaning up temporary data from old regions at 1732137300406 (+9 ms)Running coprocessor post-open hooks at 1732137300410 (+4 ms)Region opened successfully at 1732137300411 (+1 ms) 2024-11-20T21:15:00,412 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., pid=61, masterSystemTime=1732137300391 2024-11-20T21:15:00,414 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:00,414 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=61}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:00,415 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=a5436f89d60238ad7387432d34167717, regionState=OPEN, openSeqNum=2, regionLocation=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:00,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 because future has completed 2024-11-20T21:15:00,419 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-20T21:15:00,419 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 in 181 msec 2024-11-20T21:15:00,421 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-20T21:15:00,421 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5436f89d60238ad7387432d34167717, ASSIGN in 346 msec 2024-11-20T21:15:00,421 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T21:15:00,422 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732137300421"}]},"ts":"1732137300421"} 2024-11-20T21:15:00,423 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T21:15:00,424 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=59, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T21:15:00,426 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1950 sec 2024-11-20T21:15:01,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-11-20T21:15:01,377 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:15:01,380 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12616921, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:01,380 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:15:01,381 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:15:01,382 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:15:01,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:15:01,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:15:01,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f9aa12f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:01,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:15:01,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:15:01,383 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:01,384 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:15:01,384 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c78f440, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:01,385 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:15:01,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:15:01,386 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:15:01,387 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:15:01,388 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T21:15:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 0b16ab3f13b7,39687,1732137235812 2024-11-20T21:15:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@46b2bccb 2024-11-20T21:15:01,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T21:15:01,390 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52548, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T21:15:01,391 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21c1be0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:01,391 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:15:01,391 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:15:01,392 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:15:01,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:15:01,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:15:01,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@600703f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:01,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:15:01,392 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:15:01,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:01,394 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52570, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:15:01,394 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ee91c8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:01,399 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:01,399 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:15:01,399 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:15:01,399 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:15:01,400 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:15:01,400 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:15:01,400 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@239e0abf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:01,400 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:15:01,400 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:15:01,400 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:01,401 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52576, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:15:01,402 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@654dce5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:01,402 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:01,402 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:15:01,403 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:15:01,403 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:15:01,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:15:01,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:15:01,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@780fe616, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:01,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:15:01,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:15:01,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:01,405 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52594, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:15:01,405 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66ae4d2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:01,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:01,409 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:15:01,409 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:15:01,409 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:15:01,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:15:01,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:15:01,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@406e68a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:01,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:15:01,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:15:01,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:01,411 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52616, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:15:01,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1730bedc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:01,417 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:01,417 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:15:01,417 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:15:01,418 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:15:01,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:15:01,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:15:01,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3241574c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:01,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:15:01,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:15:01,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:01,419 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52634, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:15:01,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d2e267b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:01,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2856abc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:01,421 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:15:01,421 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:15:01,422 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:15:01,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:15:01,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:15:01,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@115d1551, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:01,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:15:01,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:15:01,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:01,423 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52654, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:15:01,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@542e71d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:01,425 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:01,425 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:15:01,425 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:15:01,425 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:15:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:15:01,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:15:01,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fd7cdbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:01,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:15:01,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:15:01,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:01,427 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52672, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:15:01,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c71d5d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:01,432 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a5c877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:01,432 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:15:01,432 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:15:01,433 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:15:01,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:15:01,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:15:01,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@358beaa1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:01,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:15:01,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:15:01,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:01,435 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52682, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:15:01,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@641eaec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:01,436 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679f7426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:01,437 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:15:01,437 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:15:01,437 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:15:01,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:15:01,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:15:01,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39dada81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:01,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:15:01,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:15:01,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:01,439 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:15:01,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b61b09b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:01,440 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b71af22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:01,441 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:15:01,441 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:15:01,441 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:15:01,441 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:15:01,442 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:15:01,442 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61ba118e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:01,442 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:15:01,442 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:15:01,442 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:01,443 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:52704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:15:01,444 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e3c5533, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:01,449 DEBUG [Thread-1211 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:15:01,449 DEBUG [Thread-1216 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:15:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:15:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:15:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:15:01,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:15:01,451 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:15:01,453 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60014, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:15:01,453 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2] 2024-11-20T21:15:01,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2] 2024-11-20T21:15:01,458 DEBUG [Thread-1218 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:15:01,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:15:01,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:15:01,460 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:15:01,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2] 2024-11-20T21:15:01,470 DEBUG [Thread-1220 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:15:01,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:15:01,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:15:01,472 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60034, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:15:01,472 DEBUG [Thread-1224 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:15:01,473 DEBUG [Thread-1224 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:15:01,474 DEBUG [Thread-1224 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:15:01,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2] 2024-11-20T21:15:01,475 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60044, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:15:01,475 DEBUG [Thread-1207 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:15:01,475 DEBUG [Thread-1205 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:15:01,476 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:15:01,476 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:15:01,477 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:15:01,477 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:15:01,478 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:15:01,478 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60062, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:15:01,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2] 2024-11-20T21:15:01,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2] 2024-11-20T21:15:01,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2] 2024-11-20T21:15:01,482 DEBUG [Thread-1209 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:15:01,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:15:01,483 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:15:01,484 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60070, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:15:01,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2] 2024-11-20T21:15:01,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on a5436f89d60238ad7387432d34167717 2024-11-20T21:15:01,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5436f89d60238ad7387432d34167717 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T21:15:01,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=A 2024-11-20T21:15:01,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:01,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=B 2024-11-20T21:15:01,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:01,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=C 2024-11-20T21:15:01,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:01,493 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:15:01,493 DEBUG [Thread-1213 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:15:01,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-20T21:15:01,495 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:15:01,495 DEBUG [Thread-1213 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:15:01,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-20T21:15:01,496 DEBUG [Thread-1213 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:15:01,496 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:15:01,496 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:15:01,497 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60072, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:15:01,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2] 2024-11-20T21:15:01,509 DEBUG [Thread-1222 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:15:01,509 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:15:01,510 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:15:01,511 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:60086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:15:01,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2] 2024-11-20T21:15:01,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:01,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137361512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:01,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:01,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:01,516 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:01,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:01,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137361513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:01,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:01,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:01,518 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:01,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:01,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137361514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:01,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:01,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:01,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:01,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:01,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137361515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:01,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:01,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:01,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:01,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137361516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:01,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:01,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:01,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:01,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:01,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/22eb65125c4848ca896e55d3dddf4a25 is 50, key is test_row_0/A:col10/1732137301486/Put/seqid=0 2024-11-20T21:15:01,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742087_1263 (size=14341) 2024-11-20T21:15:01,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/22eb65125c4848ca896e55d3dddf4a25 2024-11-20T21:15:01,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-20T21:15:01,607 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/becc28bd1a9d4842a618526d7ccc5437 is 50, key is test_row_0/B:col10/1732137301486/Put/seqid=0 2024-11-20T21:15:01,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742088_1264 (size=12001) 2024-11-20T21:15:01,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/becc28bd1a9d4842a618526d7ccc5437 2024-11-20T21:15:01,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137361627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:01,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:01,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:01,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:01,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:01,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137361627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:01,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:01,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:01,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137361627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:01,635 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:01,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:01,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:01,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:01,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:01,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137361627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:01,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:01,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:01,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:01,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137361627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:01,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:01,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:01,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:01,646 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:01,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-20T21:15:01,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:01,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:01,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:01,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:01,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:01,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:01,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/c6f09b7ca26542ea8f7051457fe10833 is 50, key is test_row_0/C:col10/1732137301486/Put/seqid=0 2024-11-20T21:15:01,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742089_1265 (size=12001) 2024-11-20T21:15:01,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-20T21:15:01,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:01,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:01,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:01,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:01,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:01,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:01,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-20T21:15:01,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:01,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137361849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:01,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:01,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:01,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137361849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:01,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:01,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:01,854 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:01,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:01,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137361849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:01,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:01,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:01,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:01,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137361849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:01,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:01,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:01,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:01,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137361857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:01,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:01,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:01,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:01,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-20T21:15:01,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:01,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:01,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:01,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:01,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:02,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/c6f09b7ca26542ea8f7051457fe10833 2024-11-20T21:15:02,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/22eb65125c4848ca896e55d3dddf4a25 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/22eb65125c4848ca896e55d3dddf4a25 2024-11-20T21:15:02,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-20T21:15:02,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:02,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:02,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:02,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:02,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:02,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:02,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/22eb65125c4848ca896e55d3dddf4a25, entries=200, sequenceid=14, filesize=14.0 K 2024-11-20T21:15:02,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-20T21:15:02,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/becc28bd1a9d4842a618526d7ccc5437 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/becc28bd1a9d4842a618526d7ccc5437 2024-11-20T21:15:02,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/becc28bd1a9d4842a618526d7ccc5437, entries=150, sequenceid=14, filesize=11.7 K 2024-11-20T21:15:02,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/c6f09b7ca26542ea8f7051457fe10833 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/c6f09b7ca26542ea8f7051457fe10833 2024-11-20T21:15:02,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/c6f09b7ca26542ea8f7051457fe10833, entries=150, sequenceid=14, filesize=11.7 K 2024-11-20T21:15:02,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for a5436f89d60238ad7387432d34167717 in 655ms, sequenceid=14, compaction requested=false 2024-11-20T21:15:02,145 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-20T21:15:02,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:02,166 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5436f89d60238ad7387432d34167717 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T21:15:02,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=A 2024-11-20T21:15:02,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:02,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=B 2024-11-20T21:15:02,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:02,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=C 2024-11-20T21:15:02,166 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:02,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on a5436f89d60238ad7387432d34167717 2024-11-20T21:15:02,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/971ad6c2479040448e811dad71a40756 is 50, key is test_row_0/A:col10/1732137301512/Put/seqid=0 2024-11-20T21:15:02,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:02,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137362194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:02,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:02,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:02,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137362196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:02,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:02,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137362199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:02,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:02,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:02,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137362209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:02,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:02,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:02,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137362213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:02,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:02,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742090_1266 (size=14341) 2024-11-20T21:15:02,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/971ad6c2479040448e811dad71a40756 2024-11-20T21:15:02,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:02,255 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/9083ba1e3518460c9603816495734b47 is 50, key is test_row_0/B:col10/1732137301512/Put/seqid=0 2024-11-20T21:15:02,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-20T21:15:02,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:02,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:02,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:02,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:02,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:02,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:02,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742091_1267 (size=12001) 2024-11-20T21:15:02,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/9083ba1e3518460c9603816495734b47 2024-11-20T21:15:02,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:02,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137362307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:02,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:02,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:02,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137362321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:02,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/4d51eec789444326874535f0b4b91cae is 50, key is test_row_0/C:col10/1732137301512/Put/seqid=0 2024-11-20T21:15:02,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:02,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:02,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137362329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:02,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:02,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137362329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:02,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:02,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:02,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,340 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:02,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137362337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:02,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:02,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742092_1268 (size=12001) 2024-11-20T21:15:02,350 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/4d51eec789444326874535f0b4b91cae 2024-11-20T21:15:02,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/971ad6c2479040448e811dad71a40756 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/971ad6c2479040448e811dad71a40756 2024-11-20T21:15:02,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/971ad6c2479040448e811dad71a40756, entries=200, sequenceid=34, filesize=14.0 K 2024-11-20T21:15:02,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/9083ba1e3518460c9603816495734b47 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/9083ba1e3518460c9603816495734b47 2024-11-20T21:15:02,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/9083ba1e3518460c9603816495734b47, entries=150, sequenceid=34, filesize=11.7 K 2024-11-20T21:15:02,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/4d51eec789444326874535f0b4b91cae as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/4d51eec789444326874535f0b4b91cae 2024-11-20T21:15:02,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-20T21:15:02,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:02,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:02,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:02,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:02,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:02,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:02,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/4d51eec789444326874535f0b4b91cae, entries=150, sequenceid=34, filesize=11.7 K 2024-11-20T21:15:02,443 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for a5436f89d60238ad7387432d34167717 in 277ms, sequenceid=34, compaction requested=false 2024-11-20T21:15:02,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:02,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on a5436f89d60238ad7387432d34167717 2024-11-20T21:15:02,521 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5436f89d60238ad7387432d34167717 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-20T21:15:02,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=A 2024-11-20T21:15:02,521 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:02,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=B 2024-11-20T21:15:02,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:02,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=C 2024-11-20T21:15:02,522 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:02,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/0bfb5fb575574f229c3550f3640afb7d is 50, key is test_row_0/A:col10/1732137302197/Put/seqid=0 2024-11-20T21:15:02,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-20T21:15:02,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:02,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:02,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:02,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:02,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:02,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:02,599 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:02,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137362588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:02,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:02,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137362589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:02,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,600 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:02,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:02,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:02,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742093_1269 (size=14341) 2024-11-20T21:15:02,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137362590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:02,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,604 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:02,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137362593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:02,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:02,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:02,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:02,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137362600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:02,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:02,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-20T21:15:02,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:02,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137362707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:02,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:02,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137362707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:02,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:02,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:02,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137362707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:02,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:02,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:02,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137362707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:02,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:02,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:02,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:02,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137362717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:02,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:02,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-20T21:15:02,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:02,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:02,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:02,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:02,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:02,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:02,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-20T21:15:02,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:02,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:02,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:02,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:02,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:02,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:02,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:02,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137362920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:02,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:02,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:02,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137362921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:02,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137362921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:02,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:02,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137362921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:02,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:02,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:02,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:02,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:02,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:02,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137362927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:02,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:03,005 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/0bfb5fb575574f229c3550f3640afb7d 2024-11-20T21:15:03,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/42a89dd70fae426d8b33321b321b9bcd is 50, key is test_row_0/B:col10/1732137302197/Put/seqid=0 2024-11-20T21:15:03,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742094_1270 (size=12001) 2024-11-20T21:15:03,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/42a89dd70fae426d8b33321b321b9bcd 2024-11-20T21:15:03,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/f38d2ada4ad44ec98729fe6a5c6db2eb is 50, key is test_row_0/C:col10/1732137302197/Put/seqid=0 2024-11-20T21:15:03,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-20T21:15:03,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:03,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:03,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:03,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:03,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:03,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:03,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742095_1271 (size=12001) 2024-11-20T21:15:03,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/f38d2ada4ad44ec98729fe6a5c6db2eb 2024-11-20T21:15:03,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/0bfb5fb575574f229c3550f3640afb7d as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/0bfb5fb575574f229c3550f3640afb7d 2024-11-20T21:15:03,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/0bfb5fb575574f229c3550f3640afb7d, entries=200, sequenceid=48, filesize=14.0 K 2024-11-20T21:15:03,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/42a89dd70fae426d8b33321b321b9bcd as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/42a89dd70fae426d8b33321b321b9bcd 2024-11-20T21:15:03,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/42a89dd70fae426d8b33321b321b9bcd, entries=150, sequenceid=48, filesize=11.7 K 2024-11-20T21:15:03,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/f38d2ada4ad44ec98729fe6a5c6db2eb as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/f38d2ada4ad44ec98729fe6a5c6db2eb 2024-11-20T21:15:03,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/f38d2ada4ad44ec98729fe6a5c6db2eb, entries=150, sequenceid=48, filesize=11.7 K 2024-11-20T21:15:03,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for a5436f89d60238ad7387432d34167717 in 659ms, sequenceid=48, compaction requested=true 2024-11-20T21:15:03,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:03,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:15:03,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:03,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:15:03,181 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:03,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:03,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:15:03,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T21:15:03,181 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:03,188 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:03,188 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): a5436f89d60238ad7387432d34167717/A is initiating minor compaction (all files) 2024-11-20T21:15:03,188 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5436f89d60238ad7387432d34167717/A in TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:03,188 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/22eb65125c4848ca896e55d3dddf4a25, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/971ad6c2479040448e811dad71a40756, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/0bfb5fb575574f229c3550f3640afb7d] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp, totalSize=42.0 K 2024-11-20T21:15:03,189 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:03,189 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): a5436f89d60238ad7387432d34167717/B is initiating minor compaction (all files) 2024-11-20T21:15:03,189 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5436f89d60238ad7387432d34167717/B in TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:03,189 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/becc28bd1a9d4842a618526d7ccc5437, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/9083ba1e3518460c9603816495734b47, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/42a89dd70fae426d8b33321b321b9bcd] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp, totalSize=35.2 K 2024-11-20T21:15:03,189 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 22eb65125c4848ca896e55d3dddf4a25, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732137301481 2024-11-20T21:15:03,190 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting becc28bd1a9d4842a618526d7ccc5437, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732137301486 2024-11-20T21:15:03,190 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 971ad6c2479040448e811dad71a40756, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1732137301504 2024-11-20T21:15:03,190 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9083ba1e3518460c9603816495734b47, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1732137301504 2024-11-20T21:15:03,190 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bfb5fb575574f229c3550f3640afb7d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732137302193 2024-11-20T21:15:03,191 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42a89dd70fae426d8b33321b321b9bcd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732137302193 2024-11-20T21:15:03,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-20T21:15:03,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:03,204 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing a5436f89d60238ad7387432d34167717 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-20T21:15:03,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=A 2024-11-20T21:15:03,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:03,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=B 2024-11-20T21:15:03,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:03,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=C 2024-11-20T21:15:03,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:03,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:03,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on a5436f89d60238ad7387432d34167717 2024-11-20T21:15:03,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/bd8ebb8d74e84a35bbbfb3a4222a2640 is 50, key is test_row_0/A:col10/1732137302596/Put/seqid=0 2024-11-20T21:15:03,247 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5436f89d60238ad7387432d34167717#A#compaction#220 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:03,248 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/f500248ec1d341808dc02849068d0d55 is 50, key is test_row_0/A:col10/1732137302197/Put/seqid=0 2024-11-20T21:15:03,254 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5436f89d60238ad7387432d34167717#B#compaction#221 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:03,254 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/7181a63626d44e91979a2be28ae61a11 is 50, key is test_row_0/B:col10/1732137302197/Put/seqid=0 2024-11-20T21:15:03,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:03,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137363259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:03,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:03,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:03,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137363261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:03,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:03,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:03,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137363263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:03,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:03,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:03,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137363265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:03,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:03,277 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:03,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137363268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:03,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:03,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742098_1274 (size=12104) 2024-11-20T21:15:03,302 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/7181a63626d44e91979a2be28ae61a11 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/7181a63626d44e91979a2be28ae61a11 2024-11-20T21:15:03,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742097_1273 (size=12104) 2024-11-20T21:15:03,313 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5436f89d60238ad7387432d34167717/B of a5436f89d60238ad7387432d34167717 into 7181a63626d44e91979a2be28ae61a11(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:03,313 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:03,313 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., storeName=a5436f89d60238ad7387432d34167717/B, priority=13, startTime=1732137303181; duration=0sec 2024-11-20T21:15:03,313 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:03,313 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:B 2024-11-20T21:15:03,313 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:03,314 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:03,314 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): a5436f89d60238ad7387432d34167717/C is initiating minor compaction (all files) 2024-11-20T21:15:03,314 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5436f89d60238ad7387432d34167717/C in TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:03,315 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/c6f09b7ca26542ea8f7051457fe10833, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/4d51eec789444326874535f0b4b91cae, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/f38d2ada4ad44ec98729fe6a5c6db2eb] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp, totalSize=35.2 K 2024-11-20T21:15:03,315 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6f09b7ca26542ea8f7051457fe10833, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732137301486 2024-11-20T21:15:03,316 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d51eec789444326874535f0b4b91cae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1732137301504 2024-11-20T21:15:03,316 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f38d2ada4ad44ec98729fe6a5c6db2eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732137302193 2024-11-20T21:15:03,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742096_1272 (size=12001) 2024-11-20T21:15:03,343 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/bd8ebb8d74e84a35bbbfb3a4222a2640 2024-11-20T21:15:03,348 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/f500248ec1d341808dc02849068d0d55 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/f500248ec1d341808dc02849068d0d55 2024-11-20T21:15:03,354 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5436f89d60238ad7387432d34167717/A of a5436f89d60238ad7387432d34167717 into f500248ec1d341808dc02849068d0d55(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:03,354 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:03,354 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., storeName=a5436f89d60238ad7387432d34167717/A, priority=13, startTime=1732137303181; duration=0sec 2024-11-20T21:15:03,354 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:03,354 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:A 2024-11-20T21:15:03,363 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5436f89d60238ad7387432d34167717#C#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:03,364 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/63ec8b5a3dda4839ba11ef05e695f09e is 50, key is test_row_0/C:col10/1732137302197/Put/seqid=0 2024-11-20T21:15:03,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:03,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137363377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:03,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:03,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137363377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137363377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,385 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:03,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137363377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:03,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,387 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:03,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:03,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137363386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:03,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:03,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742099_1275 (size=12104) 2024-11-20T21:15:03,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/72e6816f57744239819ab17ec9e0719d is 50, key is test_row_0/B:col10/1732137302596/Put/seqid=0 2024-11-20T21:15:03,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742100_1276 (size=12001) 2024-11-20T21:15:03,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:03,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137363598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:03,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:03,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:03,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137363598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:03,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:03,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:03,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137363601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:03,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:03,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:03,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137363601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:03,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:03,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:03,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137363609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:03,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-20T21:15:03,841 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/63ec8b5a3dda4839ba11ef05e695f09e as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/63ec8b5a3dda4839ba11ef05e695f09e 2024-11-20T21:15:03,870 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5436f89d60238ad7387432d34167717/C of a5436f89d60238ad7387432d34167717 into 63ec8b5a3dda4839ba11ef05e695f09e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:03,870 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:03,870 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., storeName=a5436f89d60238ad7387432d34167717/C, priority=13, startTime=1732137303181; duration=0sec 2024-11-20T21:15:03,870 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:03,870 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:C 2024-11-20T21:15:03,876 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/72e6816f57744239819ab17ec9e0719d 2024-11-20T21:15:03,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/6abe23639d1e4475894b5e928984b345 is 50, key is test_row_0/C:col10/1732137302596/Put/seqid=0 2024-11-20T21:15:03,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:03,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137363908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:03,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:03,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:03,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137363908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:03,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:03,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:03,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137363908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:03,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:03,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:03,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137363918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:03,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:03,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137363918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:03,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:03,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:03,924 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:03,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742101_1277 (size=12001) 2024-11-20T21:15:04,364 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/6abe23639d1e4475894b5e928984b345 2024-11-20T21:15:04,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/bd8ebb8d74e84a35bbbfb3a4222a2640 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/bd8ebb8d74e84a35bbbfb3a4222a2640 2024-11-20T21:15:04,378 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/bd8ebb8d74e84a35bbbfb3a4222a2640, entries=150, sequenceid=66, filesize=11.7 K 2024-11-20T21:15:04,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/72e6816f57744239819ab17ec9e0719d as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/72e6816f57744239819ab17ec9e0719d 2024-11-20T21:15:04,389 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/72e6816f57744239819ab17ec9e0719d, entries=150, sequenceid=66, filesize=11.7 K 2024-11-20T21:15:04,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/6abe23639d1e4475894b5e928984b345 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/6abe23639d1e4475894b5e928984b345 2024-11-20T21:15:04,396 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/6abe23639d1e4475894b5e928984b345, entries=150, sequenceid=66, filesize=11.7 K 2024-11-20T21:15:04,398 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for a5436f89d60238ad7387432d34167717 in 1194ms, sequenceid=66, compaction requested=false 2024-11-20T21:15:04,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:04,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:04,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-20T21:15:04,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-20T21:15:04,403 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-20T21:15:04,403 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9040 sec 2024-11-20T21:15:04,405 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 2.9100 sec 2024-11-20T21:15:04,423 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5436f89d60238ad7387432d34167717 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-20T21:15:04,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=A 2024-11-20T21:15:04,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:04,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=B 2024-11-20T21:15:04,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:04,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=C 2024-11-20T21:15:04,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:04,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on a5436f89d60238ad7387432d34167717 2024-11-20T21:15:04,427 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T21:15:04,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/483102a936394eaba2f0861c17e607fa is 50, key is test_row_0/A:col10/1732137303261/Put/seqid=0 2024-11-20T21:15:04,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742102_1278 (size=14341) 2024-11-20T21:15:04,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/483102a936394eaba2f0861c17e607fa 2024-11-20T21:15:04,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:04,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137364481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:04,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:04,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137364482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:04,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:04,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:04,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137364483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:04,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:04,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:04,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137364483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:04,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,503 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:04,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:04,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137364490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:04,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:04,504 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,505 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:04,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/31be0d6b31a84d99bccbcb80c8cd17f2 is 50, key is test_row_0/B:col10/1732137303261/Put/seqid=0 2024-11-20T21:15:04,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742103_1279 (size=12001) 2024-11-20T21:15:04,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/31be0d6b31a84d99bccbcb80c8cd17f2 2024-11-20T21:15:04,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/12961304a58e4f5bb6e55eb1802a7f9e is 50, key is test_row_0/C:col10/1732137303261/Put/seqid=0 2024-11-20T21:15:04,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742104_1280 (size=12001) 2024-11-20T21:15:04,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/12961304a58e4f5bb6e55eb1802a7f9e 2024-11-20T21:15:04,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/483102a936394eaba2f0861c17e607fa as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/483102a936394eaba2f0861c17e607fa 2024-11-20T21:15:04,596 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/483102a936394eaba2f0861c17e607fa, entries=200, sequenceid=84, filesize=14.0 K 2024-11-20T21:15:04,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/31be0d6b31a84d99bccbcb80c8cd17f2 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/31be0d6b31a84d99bccbcb80c8cd17f2 2024-11-20T21:15:04,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/31be0d6b31a84d99bccbcb80c8cd17f2, entries=150, sequenceid=84, filesize=11.7 K 2024-11-20T21:15:04,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/12961304a58e4f5bb6e55eb1802a7f9e as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/12961304a58e4f5bb6e55eb1802a7f9e 2024-11-20T21:15:04,610 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/12961304a58e4f5bb6e55eb1802a7f9e, entries=150, sequenceid=84, filesize=11.7 K 2024-11-20T21:15:04,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for a5436f89d60238ad7387432d34167717 in 188ms, sequenceid=84, compaction requested=true 2024-11-20T21:15:04,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:04,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:15:04,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:04,611 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:04,611 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:04,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:15:04,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:04,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:15:04,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:04,613 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:04,613 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): a5436f89d60238ad7387432d34167717/A is initiating minor compaction (all files) 2024-11-20T21:15:04,613 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5436f89d60238ad7387432d34167717/A in TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:04,613 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/f500248ec1d341808dc02849068d0d55, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/bd8ebb8d74e84a35bbbfb3a4222a2640, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/483102a936394eaba2f0861c17e607fa] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp, totalSize=37.5 K 2024-11-20T21:15:04,613 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f500248ec1d341808dc02849068d0d55, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732137302193 2024-11-20T21:15:04,613 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:04,614 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): a5436f89d60238ad7387432d34167717/B is initiating minor compaction (all files) 2024-11-20T21:15:04,614 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5436f89d60238ad7387432d34167717/B in TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:04,614 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/7181a63626d44e91979a2be28ae61a11, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/72e6816f57744239819ab17ec9e0719d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/31be0d6b31a84d99bccbcb80c8cd17f2] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp, totalSize=35.3 K 2024-11-20T21:15:04,614 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd8ebb8d74e84a35bbbfb3a4222a2640, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732137302558 2024-11-20T21:15:04,614 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 7181a63626d44e91979a2be28ae61a11, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732137302193 2024-11-20T21:15:04,615 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 483102a936394eaba2f0861c17e607fa, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732137303261 2024-11-20T21:15:04,615 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 72e6816f57744239819ab17ec9e0719d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732137302558 2024-11-20T21:15:04,615 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 31be0d6b31a84d99bccbcb80c8cd17f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732137303261 2024-11-20T21:15:04,624 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5436f89d60238ad7387432d34167717#A#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:04,625 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/02fa269e7dbf420aa988c4e40f1cad7c is 50, key is test_row_0/A:col10/1732137303261/Put/seqid=0 2024-11-20T21:15:04,626 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5436f89d60238ad7387432d34167717 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T21:15:04,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on a5436f89d60238ad7387432d34167717 2024-11-20T21:15:04,628 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5436f89d60238ad7387432d34167717#B#compaction#229 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:04,628 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/a39b9beb2e5b44d8bab4706e9a3aff43 is 50, key is test_row_0/B:col10/1732137303261/Put/seqid=0 2024-11-20T21:15:04,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=A 2024-11-20T21:15:04,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:04,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=B 2024-11-20T21:15:04,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:04,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=C 2024-11-20T21:15:04,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:04,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:04,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137364644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:04,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:04,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:04,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137364646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:04,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:04,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:04,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137364651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:04,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742105_1281 (size=12207) 2024-11-20T21:15:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:04,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:04,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137364652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:04,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:04,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:04,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137364662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:04,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,670 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:04,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742106_1282 (size=12207) 2024-11-20T21:15:04,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/dc960f2c82884e0585408f3bee6d0b36 is 50, key is test_row_0/A:col10/1732137304626/Put/seqid=0 2024-11-20T21:15:04,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742107_1283 (size=12001) 2024-11-20T21:15:04,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/dc960f2c82884e0585408f3bee6d0b36 2024-11-20T21:15:04,759 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/71af558105894c1db085ae4ddb905cbe is 50, key is test_row_0/B:col10/1732137304626/Put/seqid=0 2024-11-20T21:15:04,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742108_1284 (size=12001) 2024-11-20T21:15:04,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:04,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137364761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:04,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:04,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:04,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137364767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:04,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:04,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:04,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137364767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:04,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:04,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:04,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137364767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:04,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:04,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:04,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137364781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:04,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:04,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:04,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137364978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:04,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:04,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:04,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137364978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:04,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137364979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:04,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:04,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:04,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137364979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:04,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:04,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,981 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:04,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:04,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:04,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137364987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:04,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:05,075 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/02fa269e7dbf420aa988c4e40f1cad7c as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/02fa269e7dbf420aa988c4e40f1cad7c 2024-11-20T21:15:05,081 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5436f89d60238ad7387432d34167717/A of a5436f89d60238ad7387432d34167717 into 02fa269e7dbf420aa988c4e40f1cad7c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:05,081 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:05,081 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., storeName=a5436f89d60238ad7387432d34167717/A, priority=13, startTime=1732137304611; duration=0sec 2024-11-20T21:15:05,082 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:05,082 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:A 2024-11-20T21:15:05,082 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:05,083 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:05,083 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): a5436f89d60238ad7387432d34167717/C is initiating minor compaction (all files) 2024-11-20T21:15:05,083 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5436f89d60238ad7387432d34167717/C in TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:05,083 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/63ec8b5a3dda4839ba11ef05e695f09e, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/6abe23639d1e4475894b5e928984b345, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/12961304a58e4f5bb6e55eb1802a7f9e] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp, totalSize=35.3 K 2024-11-20T21:15:05,084 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63ec8b5a3dda4839ba11ef05e695f09e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1732137302193 2024-11-20T21:15:05,084 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6abe23639d1e4475894b5e928984b345, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732137302558 2024-11-20T21:15:05,089 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12961304a58e4f5bb6e55eb1802a7f9e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732137303261 2024-11-20T21:15:05,090 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/a39b9beb2e5b44d8bab4706e9a3aff43 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/a39b9beb2e5b44d8bab4706e9a3aff43 2024-11-20T21:15:05,095 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5436f89d60238ad7387432d34167717/B of a5436f89d60238ad7387432d34167717 into a39b9beb2e5b44d8bab4706e9a3aff43(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:05,095 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:05,095 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., storeName=a5436f89d60238ad7387432d34167717/B, priority=13, startTime=1732137304611; duration=0sec 2024-11-20T21:15:05,095 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:05,095 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:B 2024-11-20T21:15:05,114 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5436f89d60238ad7387432d34167717#C#compaction#232 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:05,115 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/9fe9cd08e1cc498b9ee65ea97fd83315 is 50, key is test_row_0/C:col10/1732137303261/Put/seqid=0 2024-11-20T21:15:05,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742109_1285 (size=12207) 2024-11-20T21:15:05,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/71af558105894c1db085ae4ddb905cbe 2024-11-20T21:15:05,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/45cd2ddc31e547539d6c2f2e16d848d3 is 50, key is test_row_0/C:col10/1732137304626/Put/seqid=0 2024-11-20T21:15:05,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742110_1286 (size=12001) 2024-11-20T21:15:05,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:05,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137365287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:05,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:05,289 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:05,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:05,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137365287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:05,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:05,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:05,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137365287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:05,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:05,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:05,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:05,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:05,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:05,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:05,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:05,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137365287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:05,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:05,290 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:05,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:05,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:05,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137365297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:05,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:05,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:05,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:05,548 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/9fe9cd08e1cc498b9ee65ea97fd83315 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/9fe9cd08e1cc498b9ee65ea97fd83315 2024-11-20T21:15:05,554 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5436f89d60238ad7387432d34167717/C of a5436f89d60238ad7387432d34167717 into 9fe9cd08e1cc498b9ee65ea97fd83315(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:05,554 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:05,554 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., storeName=a5436f89d60238ad7387432d34167717/C, priority=13, startTime=1732137304612; duration=0sec 2024-11-20T21:15:05,555 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:05,555 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:C 2024-11-20T21:15:05,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=105 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/45cd2ddc31e547539d6c2f2e16d848d3 2024-11-20T21:15:05,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/dc960f2c82884e0585408f3bee6d0b36 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/dc960f2c82884e0585408f3bee6d0b36 2024-11-20T21:15:05,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/dc960f2c82884e0585408f3bee6d0b36, entries=150, sequenceid=105, filesize=11.7 K 2024-11-20T21:15:05,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/71af558105894c1db085ae4ddb905cbe as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/71af558105894c1db085ae4ddb905cbe 2024-11-20T21:15:05,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/71af558105894c1db085ae4ddb905cbe, entries=150, sequenceid=105, filesize=11.7 K 2024-11-20T21:15:05,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/45cd2ddc31e547539d6c2f2e16d848d3 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/45cd2ddc31e547539d6c2f2e16d848d3 2024-11-20T21:15:05,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/45cd2ddc31e547539d6c2f2e16d848d3, entries=150, sequenceid=105, filesize=11.7 K 2024-11-20T21:15:05,642 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for a5436f89d60238ad7387432d34167717 in 1016ms, sequenceid=105, compaction requested=false 2024-11-20T21:15:05,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:05,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-20T21:15:05,657 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:15:05,657 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:15:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-20T21:15:05,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-20T21:15:05,665 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:15:05,665 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:15:05,665 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:15:05,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-20T21:15:05,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on a5436f89d60238ad7387432d34167717 2024-11-20T21:15:05,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5436f89d60238ad7387432d34167717 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-20T21:15:05,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=A 2024-11-20T21:15:05,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:05,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=B 2024-11-20T21:15:05,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:05,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=C 2024-11-20T21:15:05,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:05,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/3cdc7dd469d6495c903c833605d918a5 is 50, key is test_row_0/A:col10/1732137304649/Put/seqid=0 2024-11-20T21:15:05,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-20T21:15:05,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:05,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:05,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:05,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:05,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:05,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:05,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742111_1287 (size=14341) 2024-11-20T21:15:05,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/3cdc7dd469d6495c903c833605d918a5 2024-11-20T21:15:05,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/fb3a9e62c05b47998aeef67ece01394b is 50, key is test_row_0/B:col10/1732137304649/Put/seqid=0 2024-11-20T21:15:05,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732137305836, max timestamp=1732137305859Num uniques 859; 2024-11-20T21:15:05,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T21:15:05,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732137305836, max timestamp=1732137305859Num uniques 859; 2024-11-20T21:15:05,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T21:15:05,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732137305836, max timestamp=1732137305859Num uniques 859; 2024-11-20T21:15:05,865 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T21:15:05,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:05,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137365882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:05,892 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:05,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:05,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:05,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742112_1288 (size=12001) 2024-11-20T21:15:05,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:05,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137365888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:05,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:05,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/fb3a9e62c05b47998aeef67ece01394b 2024-11-20T21:15:05,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:05,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137365889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:05,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:05,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:05,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:05,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137365890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:05,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:05,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:05,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:05,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:05,908 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:05,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137365891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:05,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:05,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:05,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:05,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/c1ec87f4ae604f74bcfe236a20d25b9f is 50, key is test_row_0/C:col10/1732137304649/Put/seqid=0 2024-11-20T21:15:05,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742113_1289 (size=12001) 2024-11-20T21:15:05,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-20T21:15:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-20T21:15:05,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:05,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:05,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:05,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:05,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:05,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:06,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:06,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137365997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:06,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:06,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:06,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137366007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:06,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:06,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137366025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:06,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:06,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137366028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:06,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:06,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137366028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:06,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:06,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:06,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-20T21:15:06,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:06,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:06,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:06,131 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:06,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:06,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:06,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:06,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137366207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:06,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,215 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:06,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:06,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137366229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:06,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:06,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137366237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:06,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:06,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137366238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:06,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:06,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,247 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:06,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:06,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137366241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:06,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:06,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-20T21:15:06,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:06,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:06,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:06,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:06,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-20T21:15:06,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:06,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/c1ec87f4ae604f74bcfe236a20d25b9f 2024-11-20T21:15:06,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/3cdc7dd469d6495c903c833605d918a5 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/3cdc7dd469d6495c903c833605d918a5 2024-11-20T21:15:06,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/3cdc7dd469d6495c903c833605d918a5, entries=200, sequenceid=121, filesize=14.0 K 2024-11-20T21:15:06,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/fb3a9e62c05b47998aeef67ece01394b as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/fb3a9e62c05b47998aeef67ece01394b 2024-11-20T21:15:06,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/fb3a9e62c05b47998aeef67ece01394b, entries=150, sequenceid=121, filesize=11.7 K 2024-11-20T21:15:06,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/c1ec87f4ae604f74bcfe236a20d25b9f as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/c1ec87f4ae604f74bcfe236a20d25b9f 2024-11-20T21:15:06,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-20T21:15:06,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:06,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:06,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:06,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:06,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:06,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:06,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/c1ec87f4ae604f74bcfe236a20d25b9f, entries=150, sequenceid=121, filesize=11.7 K 2024-11-20T21:15:06,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=120.76 KB/123660 for a5436f89d60238ad7387432d34167717 in 643ms, sequenceid=121, compaction requested=true 2024-11-20T21:15:06,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:06,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:15:06,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:06,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:15:06,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T21:15:06,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:15:06,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-20T21:15:06,447 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:06,448 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:06,449 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:06,449 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:06,449 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): a5436f89d60238ad7387432d34167717/A is initiating minor compaction (all files) 2024-11-20T21:15:06,449 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): a5436f89d60238ad7387432d34167717/C is initiating minor compaction (all files) 2024-11-20T21:15:06,449 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5436f89d60238ad7387432d34167717/A in TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:06,449 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5436f89d60238ad7387432d34167717/C in TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:06,449 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/02fa269e7dbf420aa988c4e40f1cad7c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/dc960f2c82884e0585408f3bee6d0b36, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/3cdc7dd469d6495c903c833605d918a5] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp, totalSize=37.6 K 2024-11-20T21:15:06,449 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/9fe9cd08e1cc498b9ee65ea97fd83315, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/45cd2ddc31e547539d6c2f2e16d848d3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/c1ec87f4ae604f74bcfe236a20d25b9f] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp, totalSize=35.4 K 2024-11-20T21:15:06,449 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02fa269e7dbf420aa988c4e40f1cad7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732137303261 2024-11-20T21:15:06,449 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fe9cd08e1cc498b9ee65ea97fd83315, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732137303261 2024-11-20T21:15:06,450 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc960f2c82884e0585408f3bee6d0b36, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732137304479 2024-11-20T21:15:06,450 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 45cd2ddc31e547539d6c2f2e16d848d3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732137304479 2024-11-20T21:15:06,451 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cdc7dd469d6495c903c833605d918a5, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732137304643 2024-11-20T21:15:06,451 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting c1ec87f4ae604f74bcfe236a20d25b9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732137304643 2024-11-20T21:15:06,485 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5436f89d60238ad7387432d34167717#A#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:06,486 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/4e72287192744c80bf9436324a58bc0a is 50, key is test_row_0/A:col10/1732137304649/Put/seqid=0 2024-11-20T21:15:06,495 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5436f89d60238ad7387432d34167717#C#compaction#238 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:06,496 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/3bec09d5b4374e9dbd6747ebd7d32d7b is 50, key is test_row_0/C:col10/1732137304649/Put/seqid=0 2024-11-20T21:15:06,539 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5436f89d60238ad7387432d34167717 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-20T21:15:06,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=A 2024-11-20T21:15:06,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:06,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=B 2024-11-20T21:15:06,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:06,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=C 2024-11-20T21:15:06,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:06,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on a5436f89d60238ad7387432d34167717 2024-11-20T21:15:06,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742114_1290 (size=12309) 2024-11-20T21:15:06,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/1bc13260cc2f4bf285e65173ad1b8eaf is 50, key is test_row_0/A:col10/1732137305888/Put/seqid=0 2024-11-20T21:15:06,563 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/4e72287192744c80bf9436324a58bc0a as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/4e72287192744c80bf9436324a58bc0a 2024-11-20T21:15:06,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742115_1291 (size=12309) 2024-11-20T21:15:06,569 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5436f89d60238ad7387432d34167717/A of a5436f89d60238ad7387432d34167717 into 4e72287192744c80bf9436324a58bc0a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:06,569 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:06,569 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., storeName=a5436f89d60238ad7387432d34167717/A, priority=13, startTime=1732137306447; duration=0sec 2024-11-20T21:15:06,569 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:06,570 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:A 2024-11-20T21:15:06,570 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:06,571 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:06,571 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): a5436f89d60238ad7387432d34167717/B is initiating minor compaction (all files) 2024-11-20T21:15:06,571 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5436f89d60238ad7387432d34167717/B in TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:06,571 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/a39b9beb2e5b44d8bab4706e9a3aff43, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/71af558105894c1db085ae4ddb905cbe, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/fb3a9e62c05b47998aeef67ece01394b] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp, totalSize=35.4 K 2024-11-20T21:15:06,572 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting a39b9beb2e5b44d8bab4706e9a3aff43, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1732137303261 2024-11-20T21:15:06,572 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71af558105894c1db085ae4ddb905cbe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=105, earliestPutTs=1732137304479 2024-11-20T21:15:06,572 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb3a9e62c05b47998aeef67ece01394b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732137304643 2024-11-20T21:15:06,587 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/3bec09d5b4374e9dbd6747ebd7d32d7b as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/3bec09d5b4374e9dbd6747ebd7d32d7b 2024-11-20T21:15:06,592 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5436f89d60238ad7387432d34167717/C of a5436f89d60238ad7387432d34167717 into 3bec09d5b4374e9dbd6747ebd7d32d7b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:06,592 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:06,592 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., storeName=a5436f89d60238ad7387432d34167717/C, priority=13, startTime=1732137306447; duration=0sec 2024-11-20T21:15:06,593 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:06,593 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:C 2024-11-20T21:15:06,597 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-20T21:15:06,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:06,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:06,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:06,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:06,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:06,598 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5436f89d60238ad7387432d34167717#B#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:06,598 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/710945fc237b42e3b066ee05589b1a16 is 50, key is test_row_0/B:col10/1732137304649/Put/seqid=0 2024-11-20T21:15:06,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:06,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742116_1292 (size=14541) 2024-11-20T21:15:06,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/1bc13260cc2f4bf285e65173ad1b8eaf 2024-11-20T21:15:06,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:06,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:06,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137366583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:06,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:06,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:06,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137366585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:06,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,612 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:06,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137366582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:06,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,612 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:06,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:06,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137366611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:06,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:06,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:06,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137366610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,626 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:06,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/8dedd7ee450c404a8677b8a99412dc65 is 50, key is test_row_0/B:col10/1732137305888/Put/seqid=0 2024-11-20T21:15:06,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742117_1293 (size=12309) 2024-11-20T21:15:06,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742118_1294 (size=12151) 2024-11-20T21:15:06,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:06,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137366718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:06,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:06,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137366718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:06,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:06,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137366718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:06,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:06,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:06,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:06,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137366731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:06,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:06,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:06,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137366743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:06,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-20T21:15:06,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:06,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:06,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:06,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:06,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:06,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:06,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-20T21:15:06,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-20T21:15:06,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:06,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:06,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:06,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:06,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:06,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:06,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:06,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137366937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:06,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:06,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137366937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:06,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,954 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:06,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137366937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:06,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:06,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137366937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:06,955 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:06,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:06,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:06,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137366957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:06,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:06,968 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:07,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-20T21:15:07,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:07,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:07,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:07,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:07,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:07,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:07,079 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/710945fc237b42e3b066ee05589b1a16 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/710945fc237b42e3b066ee05589b1a16 2024-11-20T21:15:07,087 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5436f89d60238ad7387432d34167717/B of a5436f89d60238ad7387432d34167717 into 710945fc237b42e3b066ee05589b1a16(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:07,087 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:07,087 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., storeName=a5436f89d60238ad7387432d34167717/B, priority=13, startTime=1732137306447; duration=0sec 2024-11-20T21:15:07,087 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:07,087 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:B 2024-11-20T21:15:07,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/8dedd7ee450c404a8677b8a99412dc65 2024-11-20T21:15:07,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/c7ba9958767f46859bb8581a9487a238 is 50, key is test_row_0/C:col10/1732137305888/Put/seqid=0 2024-11-20T21:15:07,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742119_1295 (size=12151) 2024-11-20T21:15:07,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/c7ba9958767f46859bb8581a9487a238 2024-11-20T21:15:07,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/1bc13260cc2f4bf285e65173ad1b8eaf as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/1bc13260cc2f4bf285e65173ad1b8eaf 2024-11-20T21:15:07,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/1bc13260cc2f4bf285e65173ad1b8eaf, entries=200, sequenceid=156, filesize=14.2 K 2024-11-20T21:15:07,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/8dedd7ee450c404a8677b8a99412dc65 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/8dedd7ee450c404a8677b8a99412dc65 2024-11-20T21:15:07,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/8dedd7ee450c404a8677b8a99412dc65, entries=150, sequenceid=156, filesize=11.9 K 2024-11-20T21:15:07,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/c7ba9958767f46859bb8581a9487a238 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/c7ba9958767f46859bb8581a9487a238 2024-11-20T21:15:07,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/c7ba9958767f46859bb8581a9487a238, entries=150, sequenceid=156, filesize=11.9 K 2024-11-20T21:15:07,208 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=60.38 KB/61830 for a5436f89d60238ad7387432d34167717 in 669ms, sequenceid=156, compaction requested=false 2024-11-20T21:15:07,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:07,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-20T21:15:07,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:07,218 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing a5436f89d60238ad7387432d34167717 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T21:15:07,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=A 2024-11-20T21:15:07,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:07,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=B 2024-11-20T21:15:07,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:07,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=C 2024-11-20T21:15:07,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:07,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/7cc6d337dcb74544bffe0e935ef31170 is 50, key is test_row_0/A:col10/1732137306608/Put/seqid=0 2024-11-20T21:15:07,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on a5436f89d60238ad7387432d34167717 2024-11-20T21:15:07,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:07,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742120_1296 (size=12151) 2024-11-20T21:15:07,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:07,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137367308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:07,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:07,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137367310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:07,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:07,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:07,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137367313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:07,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:07,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:07,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137367316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:07,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:07,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137367318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:07,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:07,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:07,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137367427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:07,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:07,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:07,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137367427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:07,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:07,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137367427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:07,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:07,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:07,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137367437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:07,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:07,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:07,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137367437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:07,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,443 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:07,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:07,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137367637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:07,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:07,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:07,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137367638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:07,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:07,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137367641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:07,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:07,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:07,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137367648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:07,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:07,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:07,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137367653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:07,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:07,679 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/7cc6d337dcb74544bffe0e935ef31170 2024-11-20T21:15:07,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/2e5c49f4c88240818f19110d986e5c51 is 50, key is test_row_0/B:col10/1732137306608/Put/seqid=0 2024-11-20T21:15:07,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742121_1297 (size=12151) 2024-11-20T21:15:07,732 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/2e5c49f4c88240818f19110d986e5c51 2024-11-20T21:15:07,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/2bbc915e19f94f00aa152f5bc9c1e79b is 50, key is test_row_0/C:col10/1732137306608/Put/seqid=0 2024-11-20T21:15:07,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742122_1298 (size=12151) 2024-11-20T21:15:07,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-20T21:15:07,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:07,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137367947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:07,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:07,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137367947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:07,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:07,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:07,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:07,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137367957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:07,958 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:07,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137367957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:07,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:07,958 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,959 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:07,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:07,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137367966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:07,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:07,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:08,152 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/2bbc915e19f94f00aa152f5bc9c1e79b 2024-11-20T21:15:08,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/7cc6d337dcb74544bffe0e935ef31170 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/7cc6d337dcb74544bffe0e935ef31170 2024-11-20T21:15:08,165 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/7cc6d337dcb74544bffe0e935ef31170, entries=150, sequenceid=171, filesize=11.9 K 2024-11-20T21:15:08,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/2e5c49f4c88240818f19110d986e5c51 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/2e5c49f4c88240818f19110d986e5c51 2024-11-20T21:15:08,170 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/2e5c49f4c88240818f19110d986e5c51, entries=150, sequenceid=171, filesize=11.9 K 2024-11-20T21:15:08,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/2bbc915e19f94f00aa152f5bc9c1e79b as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/2bbc915e19f94f00aa152f5bc9c1e79b 2024-11-20T21:15:08,175 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/2bbc915e19f94f00aa152f5bc9c1e79b, entries=150, sequenceid=171, filesize=11.9 K 2024-11-20T21:15:08,176 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a5436f89d60238ad7387432d34167717 in 958ms, sequenceid=171, compaction requested=true 2024-11-20T21:15:08,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:08,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:08,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-20T21:15:08,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-20T21:15:08,181 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-20T21:15:08,181 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5130 sec 2024-11-20T21:15:08,183 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 2.5230 sec 2024-11-20T21:15:08,463 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732137307303, max timestamp=1732137308459Num uniques 859; 2024-11-20T21:15:08,463 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T21:15:08,464 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5436f89d60238ad7387432d34167717 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-11-20T21:15:08,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=A 2024-11-20T21:15:08,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:08,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=B 2024-11-20T21:15:08,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:08,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=C 2024-11-20T21:15:08,464 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:08,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on a5436f89d60238ad7387432d34167717 2024-11-20T21:15:08,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/29d3d1ac07f04dc0825ee37549dcd2a8 is 50, key is test_row_0/A:col10/1732137308459/Put/seqid=0 2024-11-20T21:15:08,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742123_1299 (size=14541) 2024-11-20T21:15:08,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/29d3d1ac07f04dc0825ee37549dcd2a8 2024-11-20T21:15:08,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:08,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137368487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:08,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:08,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/044f8fb2ac334612b3e1aa4404fa42f3 is 50, key is test_row_0/B:col10/1732137308459/Put/seqid=0 2024-11-20T21:15:08,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:08,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137368493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:08,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:08,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:08,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137368495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:08,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:08,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:08,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137368495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:08,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:08,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:08,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137368495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:08,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:08,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742124_1300 (size=12151) 2024-11-20T21:15:08,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/044f8fb2ac334612b3e1aa4404fa42f3 2024-11-20T21:15:08,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/17c6eb96724449baa7a6f75be2401ba9 is 50, key is test_row_0/C:col10/1732137308459/Put/seqid=0 2024-11-20T21:15:08,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742125_1301 (size=12151) 2024-11-20T21:15:08,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/17c6eb96724449baa7a6f75be2401ba9 2024-11-20T21:15:08,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/29d3d1ac07f04dc0825ee37549dcd2a8 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/29d3d1ac07f04dc0825ee37549dcd2a8 2024-11-20T21:15:08,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/29d3d1ac07f04dc0825ee37549dcd2a8, entries=200, sequenceid=192, filesize=14.2 K 2024-11-20T21:15:08,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/044f8fb2ac334612b3e1aa4404fa42f3 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/044f8fb2ac334612b3e1aa4404fa42f3 2024-11-20T21:15:08,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/044f8fb2ac334612b3e1aa4404fa42f3, entries=150, sequenceid=192, filesize=11.9 K 2024-11-20T21:15:08,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/17c6eb96724449baa7a6f75be2401ba9 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/17c6eb96724449baa7a6f75be2401ba9 2024-11-20T21:15:08,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/17c6eb96724449baa7a6f75be2401ba9, entries=150, sequenceid=192, filesize=11.9 K 2024-11-20T21:15:08,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~89.45 KB/91600, heapSize ~262.06 KB/268352, currentSize=93.93 KB/96180 for a5436f89d60238ad7387432d34167717 in 115ms, sequenceid=192, compaction requested=true 2024-11-20T21:15:08,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:08,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:15:08,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:08,580 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:15:08,580 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:15:08,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:15:08,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:08,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:15:08,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:08,581 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:15:08,581 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): a5436f89d60238ad7387432d34167717/B is initiating minor compaction (all files) 2024-11-20T21:15:08,581 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5436f89d60238ad7387432d34167717/B in TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:08,582 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/710945fc237b42e3b066ee05589b1a16, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/8dedd7ee450c404a8677b8a99412dc65, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/2e5c49f4c88240818f19110d986e5c51, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/044f8fb2ac334612b3e1aa4404fa42f3] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp, totalSize=47.6 K 2024-11-20T21:15:08,583 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53542 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:15:08,583 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 710945fc237b42e3b066ee05589b1a16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732137304643 2024-11-20T21:15:08,583 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): a5436f89d60238ad7387432d34167717/A is initiating minor compaction (all files) 2024-11-20T21:15:08,583 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5436f89d60238ad7387432d34167717/A in TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:08,583 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/4e72287192744c80bf9436324a58bc0a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/1bc13260cc2f4bf285e65173ad1b8eaf, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/7cc6d337dcb74544bffe0e935ef31170, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/29d3d1ac07f04dc0825ee37549dcd2a8] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp, totalSize=52.3 K 2024-11-20T21:15:08,583 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e72287192744c80bf9436324a58bc0a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732137304643 2024-11-20T21:15:08,583 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dedd7ee450c404a8677b8a99412dc65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732137305887 2024-11-20T21:15:08,584 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e5c49f4c88240818f19110d986e5c51, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732137306579 2024-11-20T21:15:08,584 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bc13260cc2f4bf285e65173ad1b8eaf, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732137305886 2024-11-20T21:15:08,584 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 044f8fb2ac334612b3e1aa4404fa42f3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732137307303 2024-11-20T21:15:08,584 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cc6d337dcb74544bffe0e935ef31170, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732137306579 2024-11-20T21:15:08,585 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29d3d1ac07f04dc0825ee37549dcd2a8, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732137307303 2024-11-20T21:15:08,600 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5436f89d60238ad7387432d34167717#B#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:08,600 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/68c9f331e2364ad89de8e68451ab0494 is 50, key is test_row_0/B:col10/1732137308459/Put/seqid=0 2024-11-20T21:15:08,606 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5436f89d60238ad7387432d34167717#A#compaction#250 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:08,607 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/57f027aea9e24e19805fc688033c891a is 50, key is test_row_0/A:col10/1732137308459/Put/seqid=0 2024-11-20T21:15:08,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5436f89d60238ad7387432d34167717 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-20T21:15:08,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on a5436f89d60238ad7387432d34167717 2024-11-20T21:15:08,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=A 2024-11-20T21:15:08,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:08,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=B 2024-11-20T21:15:08,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:08,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=C 2024-11-20T21:15:08,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:08,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742126_1302 (size=12595) 2024-11-20T21:15:08,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:08,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137368638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:08,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:08,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:08,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137368640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:08,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:08,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:08,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137368645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:08,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:08,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:08,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:08,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137368651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:08,663 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/5bf4f451f2eb4a308f5321e265f6578d is 50, key is test_row_0/A:col10/1732137308613/Put/seqid=0 2024-11-20T21:15:08,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:08,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137368651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:08,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:08,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742127_1303 (size=12595) 2024-11-20T21:15:08,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742128_1304 (size=12151) 2024-11-20T21:15:08,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/5bf4f451f2eb4a308f5321e265f6578d 2024-11-20T21:15:08,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/4971a6f9423c4ef6bfe9178b4f54c8b8 is 50, key is test_row_0/B:col10/1732137308613/Put/seqid=0 2024-11-20T21:15:08,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742129_1305 (size=12151) 2024-11-20T21:15:08,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:08,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137368758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:08,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:08,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:08,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137368758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:08,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:08,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:08,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137368758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:08,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:08,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:08,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137368769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:08,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:08,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:08,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137368769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:08,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:08,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137368967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:08,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137368968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:08,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:08,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137368968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:08,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:08,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:08,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137368977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:08,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:08,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137368977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:09,059 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/68c9f331e2364ad89de8e68451ab0494 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/68c9f331e2364ad89de8e68451ab0494 2024-11-20T21:15:09,066 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a5436f89d60238ad7387432d34167717/B of a5436f89d60238ad7387432d34167717 into 68c9f331e2364ad89de8e68451ab0494(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:09,066 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:09,066 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., storeName=a5436f89d60238ad7387432d34167717/B, priority=12, startTime=1732137308580; duration=0sec 2024-11-20T21:15:09,067 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:09,067 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:B 2024-11-20T21:15:09,067 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:15:09,069 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48762 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:15:09,069 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): a5436f89d60238ad7387432d34167717/C is initiating minor compaction (all files) 2024-11-20T21:15:09,069 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5436f89d60238ad7387432d34167717/C in TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:09,070 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/3bec09d5b4374e9dbd6747ebd7d32d7b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/c7ba9958767f46859bb8581a9487a238, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/2bbc915e19f94f00aa152f5bc9c1e79b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/17c6eb96724449baa7a6f75be2401ba9] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp, totalSize=47.6 K 2024-11-20T21:15:09,070 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bec09d5b4374e9dbd6747ebd7d32d7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1732137304643 2024-11-20T21:15:09,070 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting c7ba9958767f46859bb8581a9487a238, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732137305887 2024-11-20T21:15:09,071 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bbc915e19f94f00aa152f5bc9c1e79b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1732137306579 2024-11-20T21:15:09,071 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 17c6eb96724449baa7a6f75be2401ba9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732137307303 2024-11-20T21:15:09,082 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5436f89d60238ad7387432d34167717#C#compaction#253 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:09,083 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/1cd33031772444c68e6579d7b356ec12 is 50, key is test_row_0/C:col10/1732137308459/Put/seqid=0 2024-11-20T21:15:09,095 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/57f027aea9e24e19805fc688033c891a as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/57f027aea9e24e19805fc688033c891a 2024-11-20T21:15:09,101 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a5436f89d60238ad7387432d34167717/A of a5436f89d60238ad7387432d34167717 into 57f027aea9e24e19805fc688033c891a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:09,101 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:09,101 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., storeName=a5436f89d60238ad7387432d34167717/A, priority=12, startTime=1732137308580; duration=0sec 2024-11-20T21:15:09,101 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:09,101 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:A 2024-11-20T21:15:09,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742130_1306 (size=12595) 2024-11-20T21:15:09,110 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/1cd33031772444c68e6579d7b356ec12 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/1cd33031772444c68e6579d7b356ec12 2024-11-20T21:15:09,114 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a5436f89d60238ad7387432d34167717/C of a5436f89d60238ad7387432d34167717 into 1cd33031772444c68e6579d7b356ec12(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:09,114 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:09,114 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., storeName=a5436f89d60238ad7387432d34167717/C, priority=12, startTime=1732137308580; duration=0sec 2024-11-20T21:15:09,115 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:09,115 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:C 2024-11-20T21:15:09,134 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/4971a6f9423c4ef6bfe9178b4f54c8b8 2024-11-20T21:15:09,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/db9909c20c4947af8fd9873dd8d53657 is 50, key is test_row_0/C:col10/1732137308613/Put/seqid=0 2024-11-20T21:15:09,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742131_1307 (size=12151) 2024-11-20T21:15:09,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/db9909c20c4947af8fd9873dd8d53657 2024-11-20T21:15:09,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/5bf4f451f2eb4a308f5321e265f6578d as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/5bf4f451f2eb4a308f5321e265f6578d 2024-11-20T21:15:09,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/5bf4f451f2eb4a308f5321e265f6578d, entries=150, sequenceid=212, filesize=11.9 K 2024-11-20T21:15:09,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/4971a6f9423c4ef6bfe9178b4f54c8b8 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/4971a6f9423c4ef6bfe9178b4f54c8b8 2024-11-20T21:15:09,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/4971a6f9423c4ef6bfe9178b4f54c8b8, entries=150, sequenceid=212, filesize=11.9 K 2024-11-20T21:15:09,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/db9909c20c4947af8fd9873dd8d53657 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/db9909c20c4947af8fd9873dd8d53657 2024-11-20T21:15:09,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/db9909c20c4947af8fd9873dd8d53657, entries=150, sequenceid=212, filesize=11.9 K 2024-11-20T21:15:09,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for a5436f89d60238ad7387432d34167717 in 558ms, sequenceid=212, compaction requested=false 2024-11-20T21:15:09,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:09,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on a5436f89d60238ad7387432d34167717 2024-11-20T21:15:09,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5436f89d60238ad7387432d34167717 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-20T21:15:09,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=A 2024-11-20T21:15:09,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:09,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=B 2024-11-20T21:15:09,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:09,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=C 2024-11-20T21:15:09,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:09,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/8fa5706069e3468a97e144e30f37cb31 is 50, key is test_row_0/A:col10/1732137309279/Put/seqid=0 2024-11-20T21:15:09,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742132_1308 (size=16931) 2024-11-20T21:15:09,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137369301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:09,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137369303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:09,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:09,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:09,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:09,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137369307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:09,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:09,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:09,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137369308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:09,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:09,314 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:09,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137369309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:09,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,315 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:09,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137369418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:09,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:09,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137369418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:09,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137369418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:09,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:09,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137369418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:09,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,422 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:09,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:09,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:09,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137369419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:09,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:09,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:09,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,423 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:09,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137369627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:09,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:09,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137369627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:09,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:09,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137369627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:09,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:09,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:09,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:09,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137369627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:09,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137369628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:09,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,633 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:09,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:09,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/8fa5706069e3468a97e144e30f37cb31 2024-11-20T21:15:09,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/5479bf73d5344b67ba50f205b56f0d81 is 50, key is test_row_0/B:col10/1732137309279/Put/seqid=0 2024-11-20T21:15:09,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742133_1309 (size=12151) 2024-11-20T21:15:09,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/5479bf73d5344b67ba50f205b56f0d81 2024-11-20T21:15:09,779 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/dd26f54c1f3b470fb13c7aed72066b56 is 50, key is test_row_0/C:col10/1732137309279/Put/seqid=0 2024-11-20T21:15:09,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742134_1310 (size=12151) 2024-11-20T21:15:09,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-20T21:15:09,817 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:15:09,820 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:15:09,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-20T21:15:09,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T21:15:09,823 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:15:09,824 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:15:09,824 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:15:09,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T21:15:09,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:09,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137369941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:09,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,947 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:09,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:09,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137369941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:09,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:09,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137369943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:09,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:09,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,948 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:09,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:09,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137369945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:09,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:09,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:09,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137369950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:09,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:09,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:09,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T21:15:09,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:09,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:09,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:09,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:09,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:09,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:10,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T21:15:10,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:10,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:10,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:10,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:10,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:10,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T21:15:10,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:10,195 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/dd26f54c1f3b470fb13c7aed72066b56 2024-11-20T21:15:10,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/8fa5706069e3468a97e144e30f37cb31 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/8fa5706069e3468a97e144e30f37cb31 2024-11-20T21:15:10,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/8fa5706069e3468a97e144e30f37cb31, entries=250, sequenceid=231, filesize=16.5 K 2024-11-20T21:15:10,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/5479bf73d5344b67ba50f205b56f0d81 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/5479bf73d5344b67ba50f205b56f0d81 2024-11-20T21:15:10,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/5479bf73d5344b67ba50f205b56f0d81, entries=150, sequenceid=231, filesize=11.9 K 2024-11-20T21:15:10,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/dd26f54c1f3b470fb13c7aed72066b56 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/dd26f54c1f3b470fb13c7aed72066b56 2024-11-20T21:15:10,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/dd26f54c1f3b470fb13c7aed72066b56, entries=150, sequenceid=231, filesize=11.9 K 2024-11-20T21:15:10,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for a5436f89d60238ad7387432d34167717 in 939ms, sequenceid=231, compaction requested=true 2024-11-20T21:15:10,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:10,221 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:10,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:15:10,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:10,221 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:10,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:15:10,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:10,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:15:10,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:10,227 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:10,227 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): a5436f89d60238ad7387432d34167717/A is initiating minor compaction (all files) 2024-11-20T21:15:10,227 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5436f89d60238ad7387432d34167717/A in TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:10,227 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/57f027aea9e24e19805fc688033c891a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/5bf4f451f2eb4a308f5321e265f6578d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/8fa5706069e3468a97e144e30f37cb31] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp, totalSize=40.7 K 2024-11-20T21:15:10,227 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:10,227 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): a5436f89d60238ad7387432d34167717/B is initiating minor compaction (all files) 2024-11-20T21:15:10,227 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5436f89d60238ad7387432d34167717/B in TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:10,227 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/68c9f331e2364ad89de8e68451ab0494, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/4971a6f9423c4ef6bfe9178b4f54c8b8, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/5479bf73d5344b67ba50f205b56f0d81] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp, totalSize=36.0 K 2024-11-20T21:15:10,232 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57f027aea9e24e19805fc688033c891a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732137307303 2024-11-20T21:15:10,233 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 68c9f331e2364ad89de8e68451ab0494, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732137307303 2024-11-20T21:15:10,233 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5bf4f451f2eb4a308f5321e265f6578d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732137308485 2024-11-20T21:15:10,244 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 4971a6f9423c4ef6bfe9178b4f54c8b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732137308485 2024-11-20T21:15:10,244 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fa5706069e3468a97e144e30f37cb31, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732137308649 2024-11-20T21:15:10,246 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 5479bf73d5344b67ba50f205b56f0d81, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732137308649 2024-11-20T21:15:10,287 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5436f89d60238ad7387432d34167717#B#compaction#258 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:10,287 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/5a694f8130f74b15a62a55c720a43a10 is 50, key is test_row_0/B:col10/1732137309279/Put/seqid=0 2024-11-20T21:15:10,295 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5436f89d60238ad7387432d34167717#A#compaction#259 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:10,296 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/e0675079aade4e0e950459fe05aeef10 is 50, key is test_row_0/A:col10/1732137309279/Put/seqid=0 2024-11-20T21:15:10,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-20T21:15:10,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:10,296 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing a5436f89d60238ad7387432d34167717 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T21:15:10,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=A 2024-11-20T21:15:10,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:10,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=B 2024-11-20T21:15:10,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:10,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=C 2024-11-20T21:15:10,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:10,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/c6ebeb145a194c00b379c2d0f4f9a18d is 50, key is test_row_0/A:col10/1732137309305/Put/seqid=0 2024-11-20T21:15:10,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742135_1311 (size=12697) 2024-11-20T21:15:10,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742136_1312 (size=12697) 2024-11-20T21:15:10,385 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/e0675079aade4e0e950459fe05aeef10 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/e0675079aade4e0e950459fe05aeef10 2024-11-20T21:15:10,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742137_1313 (size=12151) 2024-11-20T21:15:10,394 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/c6ebeb145a194c00b379c2d0f4f9a18d 2024-11-20T21:15:10,399 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5436f89d60238ad7387432d34167717/A of a5436f89d60238ad7387432d34167717 into e0675079aade4e0e950459fe05aeef10(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:10,399 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:10,399 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., storeName=a5436f89d60238ad7387432d34167717/A, priority=13, startTime=1732137310221; duration=0sec 2024-11-20T21:15:10,399 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:10,399 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:A 2024-11-20T21:15:10,399 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:10,402 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:10,402 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): a5436f89d60238ad7387432d34167717/C is initiating minor compaction (all files) 2024-11-20T21:15:10,402 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5436f89d60238ad7387432d34167717/C in TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:10,402 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/1cd33031772444c68e6579d7b356ec12, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/db9909c20c4947af8fd9873dd8d53657, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/dd26f54c1f3b470fb13c7aed72066b56] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp, totalSize=36.0 K 2024-11-20T21:15:10,404 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cd33031772444c68e6579d7b356ec12, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1732137307303 2024-11-20T21:15:10,406 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting db9909c20c4947af8fd9873dd8d53657, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1732137308485 2024-11-20T21:15:10,416 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd26f54c1f3b470fb13c7aed72066b56, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732137308649 2024-11-20T21:15:10,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/0fe2d34cabc0420b8d05b344b6f6e2c5 is 50, key is test_row_0/B:col10/1732137309305/Put/seqid=0 2024-11-20T21:15:10,428 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5436f89d60238ad7387432d34167717#C#compaction#262 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:10,429 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/ab62023f38284f65a9ba65b306c0dc71 is 50, key is test_row_0/C:col10/1732137309279/Put/seqid=0 2024-11-20T21:15:10,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T21:15:10,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742138_1314 (size=12151) 2024-11-20T21:15:10,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on a5436f89d60238ad7387432d34167717 2024-11-20T21:15:10,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:10,463 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/0fe2d34cabc0420b8d05b344b6f6e2c5 2024-11-20T21:15:10,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/a78bb1acdba04ca3ad54f0202ee124df is 50, key is test_row_0/C:col10/1732137309305/Put/seqid=0 2024-11-20T21:15:10,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742139_1315 (size=12697) 2024-11-20T21:15:10,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:10,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137370483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:10,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:10,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137370485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:10,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:10,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:10,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137370494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:10,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:10,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137370494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:10,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:10,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:10,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,496 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:10,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137370494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:10,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:10,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:10,502 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/ab62023f38284f65a9ba65b306c0dc71 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/ab62023f38284f65a9ba65b306c0dc71 2024-11-20T21:15:10,510 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5436f89d60238ad7387432d34167717/C of a5436f89d60238ad7387432d34167717 into ab62023f38284f65a9ba65b306c0dc71(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:10,510 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:10,511 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., storeName=a5436f89d60238ad7387432d34167717/C, priority=13, startTime=1732137310221; duration=0sec 2024-11-20T21:15:10,511 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:10,511 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:C 2024-11-20T21:15:10,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742140_1316 (size=12151) 2024-11-20T21:15:10,516 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/a78bb1acdba04ca3ad54f0202ee124df 2024-11-20T21:15:10,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/c6ebeb145a194c00b379c2d0f4f9a18d as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/c6ebeb145a194c00b379c2d0f4f9a18d 2024-11-20T21:15:10,525 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/c6ebeb145a194c00b379c2d0f4f9a18d, entries=150, sequenceid=248, filesize=11.9 K 2024-11-20T21:15:10,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/0fe2d34cabc0420b8d05b344b6f6e2c5 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/0fe2d34cabc0420b8d05b344b6f6e2c5 2024-11-20T21:15:10,531 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/0fe2d34cabc0420b8d05b344b6f6e2c5, entries=150, sequenceid=248, filesize=11.9 K 2024-11-20T21:15:10,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/a78bb1acdba04ca3ad54f0202ee124df as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/a78bb1acdba04ca3ad54f0202ee124df 2024-11-20T21:15:10,537 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/a78bb1acdba04ca3ad54f0202ee124df, entries=150, sequenceid=248, filesize=11.9 K 2024-11-20T21:15:10,540 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for a5436f89d60238ad7387432d34167717 in 244ms, sequenceid=248, compaction requested=false 2024-11-20T21:15:10,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:10,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:10,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-20T21:15:10,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-20T21:15:10,544 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-20T21:15:10,544 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 718 msec 2024-11-20T21:15:10,548 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 725 msec 2024-11-20T21:15:10,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on a5436f89d60238ad7387432d34167717 2024-11-20T21:15:10,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5436f89d60238ad7387432d34167717 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T21:15:10,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=A 2024-11-20T21:15:10,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:10,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=B 2024-11-20T21:15:10,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:10,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=C 2024-11-20T21:15:10,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:10,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/dadb85f6d6994133899ef6295aa17f82 is 50, key is test_row_0/A:col10/1732137310491/Put/seqid=0 2024-11-20T21:15:10,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742141_1317 (size=12301) 2024-11-20T21:15:10,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/dadb85f6d6994133899ef6295aa17f82 2024-11-20T21:15:10,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:10,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137370634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:10,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:10,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137370636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:10,645 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:10,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137370633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:10,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:10,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:10,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,647 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:10,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:10,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137370645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:10,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:10,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/b27391d81df84f1a808bacf0490dfeae is 50, key is test_row_0/B:col10/1732137310491/Put/seqid=0 2024-11-20T21:15:10,653 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:10,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137370647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:10,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:10,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742142_1318 (size=12301) 2024-11-20T21:15:10,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/b27391d81df84f1a808bacf0490dfeae 2024-11-20T21:15:10,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/b2c1660c349e4e0aaec83995d0afb421 is 50, key is test_row_0/C:col10/1732137310491/Put/seqid=0 2024-11-20T21:15:10,750 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/5a694f8130f74b15a62a55c720a43a10 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/5a694f8130f74b15a62a55c720a43a10 2024-11-20T21:15:10,759 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5436f89d60238ad7387432d34167717/B of a5436f89d60238ad7387432d34167717 into 5a694f8130f74b15a62a55c720a43a10(size=12.4 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:10,759 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:10,759 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., storeName=a5436f89d60238ad7387432d34167717/B, priority=13, startTime=1732137310221; duration=0sec 2024-11-20T21:15:10,760 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:10,760 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:B 2024-11-20T21:15:10,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:10,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137370757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:10,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:10,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:10,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137370757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:10,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:10,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:10,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137370758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:10,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:10,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,769 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:10,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137370758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:10,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:10,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:10,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137370759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:10,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:10,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742143_1319 (size=12301) 2024-11-20T21:15:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-20T21:15:10,957 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:15:10,958 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:15:10,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-20T21:15:10,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T21:15:10,963 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:15:10,964 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:15:10,964 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:15:10,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:10,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137370967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:10,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:10,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:10,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137370977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:10,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:10,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137370977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:10,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:10,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137370980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:10,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:10,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137370981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:10,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:10,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:10,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:10,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:10,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:11,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T21:15:11,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-20T21:15:11,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:11,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:11,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:11,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:11,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:11,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:11,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/b2c1660c349e4e0aaec83995d0afb421 2024-11-20T21:15:11,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/dadb85f6d6994133899ef6295aa17f82 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/dadb85f6d6994133899ef6295aa17f82 2024-11-20T21:15:11,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/dadb85f6d6994133899ef6295aa17f82, entries=150, sequenceid=266, filesize=12.0 K 2024-11-20T21:15:11,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/b27391d81df84f1a808bacf0490dfeae as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/b27391d81df84f1a808bacf0490dfeae 2024-11-20T21:15:11,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/b27391d81df84f1a808bacf0490dfeae, entries=150, sequenceid=266, filesize=12.0 K 2024-11-20T21:15:11,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-20T21:15:11,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:11,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/b2c1660c349e4e0aaec83995d0afb421 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/b2c1660c349e4e0aaec83995d0afb421 2024-11-20T21:15:11,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:11,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:11,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:11,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:11,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/b2c1660c349e4e0aaec83995d0afb421, entries=150, sequenceid=266, filesize=12.0 K 2024-11-20T21:15:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T21:15:11,284 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for a5436f89d60238ad7387432d34167717 in 679ms, sequenceid=266, compaction requested=true 2024-11-20T21:15:11,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:11,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:15:11,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:11,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:15:11,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T21:15:11,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:15:11,284 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-20T21:15:11,284 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:11,286 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:11,286 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): a5436f89d60238ad7387432d34167717/A is initiating minor compaction (all files) 2024-11-20T21:15:11,286 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5436f89d60238ad7387432d34167717/A in TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:11,286 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/e0675079aade4e0e950459fe05aeef10, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/c6ebeb145a194c00b379c2d0f4f9a18d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/dadb85f6d6994133899ef6295aa17f82] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp, totalSize=36.3 K 2024-11-20T21:15:11,287 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0675079aade4e0e950459fe05aeef10, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732137308649 2024-11-20T21:15:11,287 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6ebeb145a194c00b379c2d0f4f9a18d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732137309297 2024-11-20T21:15:11,288 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting dadb85f6d6994133899ef6295aa17f82, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732137310484 2024-11-20T21:15:11,288 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:11,305 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:11,305 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): a5436f89d60238ad7387432d34167717/C is initiating minor compaction (all files) 2024-11-20T21:15:11,305 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5436f89d60238ad7387432d34167717/C in TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:11,305 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/ab62023f38284f65a9ba65b306c0dc71, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/a78bb1acdba04ca3ad54f0202ee124df, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/b2c1660c349e4e0aaec83995d0afb421] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp, totalSize=36.3 K 2024-11-20T21:15:11,307 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting ab62023f38284f65a9ba65b306c0dc71, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732137308649 2024-11-20T21:15:11,308 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting a78bb1acdba04ca3ad54f0202ee124df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732137309297 2024-11-20T21:15:11,313 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting b2c1660c349e4e0aaec83995d0afb421, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732137310484 2024-11-20T21:15:11,338 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5436f89d60238ad7387432d34167717#A#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:11,338 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/3899816d1ef246288a2896b9be789274 is 50, key is test_row_0/A:col10/1732137310491/Put/seqid=0 2024-11-20T21:15:11,363 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5436f89d60238ad7387432d34167717#C#compaction#268 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:11,363 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/590d5a27f472465fab219b854ca82256 is 50, key is test_row_0/C:col10/1732137310491/Put/seqid=0 2024-11-20T21:15:11,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742144_1320 (size=12949) 2024-11-20T21:15:11,395 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/3899816d1ef246288a2896b9be789274 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/3899816d1ef246288a2896b9be789274 2024-11-20T21:15:11,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742145_1321 (size=12949) 2024-11-20T21:15:11,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on a5436f89d60238ad7387432d34167717 2024-11-20T21:15:11,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5436f89d60238ad7387432d34167717 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-20T21:15:11,408 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5436f89d60238ad7387432d34167717/A of a5436f89d60238ad7387432d34167717 into 3899816d1ef246288a2896b9be789274(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:11,408 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:11,408 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., storeName=a5436f89d60238ad7387432d34167717/A, priority=13, startTime=1732137311284; duration=0sec 2024-11-20T21:15:11,408 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:11,408 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:A 2024-11-20T21:15:11,409 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:11,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=A 2024-11-20T21:15:11,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:11,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=B 2024-11-20T21:15:11,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:11,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=C 2024-11-20T21:15:11,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:11,419 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/590d5a27f472465fab219b854ca82256 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/590d5a27f472465fab219b854ca82256 2024-11-20T21:15:11,428 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:11,429 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): a5436f89d60238ad7387432d34167717/B is initiating minor compaction (all files) 2024-11-20T21:15:11,429 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5436f89d60238ad7387432d34167717/B in TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:11,429 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/5a694f8130f74b15a62a55c720a43a10, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/0fe2d34cabc0420b8d05b344b6f6e2c5, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/b27391d81df84f1a808bacf0490dfeae] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp, totalSize=36.3 K 2024-11-20T21:15:11,431 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/744d6a6172814b42b4d91d3588dd2b3d is 50, key is test_row_0/A:col10/1732137311305/Put/seqid=0 2024-11-20T21:15:11,432 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a694f8130f74b15a62a55c720a43a10, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1732137308649 2024-11-20T21:15:11,432 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fe2d34cabc0420b8d05b344b6f6e2c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1732137309297 2024-11-20T21:15:11,433 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting b27391d81df84f1a808bacf0490dfeae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732137310484 2024-11-20T21:15:11,434 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5436f89d60238ad7387432d34167717/C of a5436f89d60238ad7387432d34167717 into 590d5a27f472465fab219b854ca82256(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:11,434 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:11,434 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., storeName=a5436f89d60238ad7387432d34167717/C, priority=13, startTime=1732137311284; duration=0sec 2024-11-20T21:15:11,434 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:11,434 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:C 2024-11-20T21:15:11,443 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5436f89d60238ad7387432d34167717#B#compaction#270 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:11,443 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/f34b88505c30493185e32880d6d72792 is 50, key is test_row_0/B:col10/1732137310491/Put/seqid=0 2024-11-20T21:15:11,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-20T21:15:11,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:11,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:11,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:11,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:11,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:11,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:11,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742146_1322 (size=12301) 2024-11-20T21:15:11,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/744d6a6172814b42b4d91d3588dd2b3d 2024-11-20T21:15:11,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:11,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137371433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:11,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:11,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:11,469 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:11,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:11,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137371442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:11,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:11,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:11,473 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:11,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:11,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137371459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:11,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:11,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:11,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:11,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:11,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137371465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:11,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:11,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:11,475 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:11,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:11,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137371469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:11,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:11,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:11,480 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:11,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742147_1323 (size=12949) 2024-11-20T21:15:11,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/089221c56d954ca4a4e7602ca7920774 is 50, key is test_row_0/B:col10/1732137311305/Put/seqid=0 2024-11-20T21:15:11,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742148_1324 (size=12301) 2024-11-20T21:15:11,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/089221c56d954ca4a4e7602ca7920774 2024-11-20T21:15:11,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/93134d06f1c144ad9dd978840ab74677 is 50, key is test_row_0/C:col10/1732137311305/Put/seqid=0 2024-11-20T21:15:11,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742149_1325 (size=12301) 2024-11-20T21:15:11,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/93134d06f1c144ad9dd978840ab74677 2024-11-20T21:15:11,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/744d6a6172814b42b4d91d3588dd2b3d as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/744d6a6172814b42b4d91d3588dd2b3d 2024-11-20T21:15:11,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/744d6a6172814b42b4d91d3588dd2b3d, entries=150, sequenceid=289, filesize=12.0 K 2024-11-20T21:15:11,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/089221c56d954ca4a4e7602ca7920774 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/089221c56d954ca4a4e7602ca7920774 2024-11-20T21:15:11,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/089221c56d954ca4a4e7602ca7920774, entries=150, sequenceid=289, filesize=12.0 K 2024-11-20T21:15:11,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/93134d06f1c144ad9dd978840ab74677 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/93134d06f1c144ad9dd978840ab74677 2024-11-20T21:15:11,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:11,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137371580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:11,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:11,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:11,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:11,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:11,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137371581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:11,583 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:11,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137371581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:11,584 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:11,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137371581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:11,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:11,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:11,584 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:11,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:11,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:11,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:11,585 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:11,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:11,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137371587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:11,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:11,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:11,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:11,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/93134d06f1c144ad9dd978840ab74677, entries=150, sequenceid=289, filesize=12.0 K 2024-11-20T21:15:11,598 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for a5436f89d60238ad7387432d34167717 in 190ms, sequenceid=289, compaction requested=false 2024-11-20T21:15:11,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:11,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T21:15:11,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-20T21:15:11,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:11,603 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing a5436f89d60238ad7387432d34167717 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T21:15:11,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=A 2024-11-20T21:15:11,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:11,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=B 2024-11-20T21:15:11,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:11,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=C 2024-11-20T21:15:11,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:11,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/b0afda355419416d8d1841a3b19a9b1f is 50, key is test_row_0/A:col10/1732137311427/Put/seqid=0 2024-11-20T21:15:11,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742150_1326 (size=12301) 2024-11-20T21:15:11,648 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/b0afda355419416d8d1841a3b19a9b1f 2024-11-20T21:15:11,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/0c6c7a3d7c404d629a7faf41cd177f3b is 50, key is test_row_0/B:col10/1732137311427/Put/seqid=0 2024-11-20T21:15:11,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742151_1327 (size=12301) 2024-11-20T21:15:11,728 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/0c6c7a3d7c404d629a7faf41cd177f3b 2024-11-20T21:15:11,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/8d5a959491ea4b7d96086d01bad28945 is 50, key is test_row_0/C:col10/1732137311427/Put/seqid=0 2024-11-20T21:15:11,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:11,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on a5436f89d60238ad7387432d34167717 2024-11-20T21:15:11,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742152_1328 (size=12301) 2024-11-20T21:15:11,798 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/8d5a959491ea4b7d96086d01bad28945 2024-11-20T21:15:11,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/b0afda355419416d8d1841a3b19a9b1f as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/b0afda355419416d8d1841a3b19a9b1f 2024-11-20T21:15:11,811 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/b0afda355419416d8d1841a3b19a9b1f, entries=150, sequenceid=302, filesize=12.0 K 2024-11-20T21:15:11,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/0c6c7a3d7c404d629a7faf41cd177f3b as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/0c6c7a3d7c404d629a7faf41cd177f3b 2024-11-20T21:15:11,819 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/0c6c7a3d7c404d629a7faf41cd177f3b, entries=150, sequenceid=302, filesize=12.0 K 2024-11-20T21:15:11,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/8d5a959491ea4b7d96086d01bad28945 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/8d5a959491ea4b7d96086d01bad28945 2024-11-20T21:15:11,833 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/8d5a959491ea4b7d96086d01bad28945, entries=150, sequenceid=302, filesize=12.0 K 2024-11-20T21:15:11,834 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=80.51 KB/82440 for a5436f89d60238ad7387432d34167717 in 231ms, sequenceid=302, compaction requested=true 2024-11-20T21:15:11,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:11,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:11,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-20T21:15:11,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-20T21:15:11,837 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-20T21:15:11,837 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 871 msec 2024-11-20T21:15:11,838 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 879 msec 2024-11-20T21:15:11,841 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5436f89d60238ad7387432d34167717 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T21:15:11,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on a5436f89d60238ad7387432d34167717 2024-11-20T21:15:11,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=A 2024-11-20T21:15:11,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:11,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=B 2024-11-20T21:15:11,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:11,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=C 2024-11-20T21:15:11,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:11,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/96d9090a665e47cf95fdc40ab40e1704 is 50, key is test_row_0/A:col10/1732137311802/Put/seqid=0 2024-11-20T21:15:11,891 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/f34b88505c30493185e32880d6d72792 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/f34b88505c30493185e32880d6d72792 2024-11-20T21:15:11,896 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5436f89d60238ad7387432d34167717/B of a5436f89d60238ad7387432d34167717 into f34b88505c30493185e32880d6d72792(size=12.6 K), total size for store is 36.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:11,896 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:11,896 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., storeName=a5436f89d60238ad7387432d34167717/B, priority=13, startTime=1732137311284; duration=0sec 2024-11-20T21:15:11,896 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:11,896 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:B 2024-11-20T21:15:11,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137371889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:11,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742153_1329 (size=14741) 2024-11-20T21:15:11,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:11,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:11,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:11,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:11,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137371889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:11,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:11,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:11,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:11,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:11,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137371891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:11,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:11,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137371892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:11,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:11,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:11,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:11,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:11,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:11,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:11,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/96d9090a665e47cf95fdc40ab40e1704 2024-11-20T21:15:11,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:11,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137371909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:11,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:11,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:11,926 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:11,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/64a8353ff03a4a55a52ee390aca73aff is 50, key is test_row_0/B:col10/1732137311802/Put/seqid=0 2024-11-20T21:15:12,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742154_1330 (size=12301) 2024-11-20T21:15:12,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/64a8353ff03a4a55a52ee390aca73aff 2024-11-20T21:15:12,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:12,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137372017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:12,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:12,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:12,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137372017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:12,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137372017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:12,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:12,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137372017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:12,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,029 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:12,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/965f0f041a304aadaf64caf287452253 is 50, key is test_row_0/C:col10/1732137311802/Put/seqid=0 2024-11-20T21:15:12,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:12,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137372037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:12,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,042 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:12,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742155_1331 (size=12301) 2024-11-20T21:15:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-20T21:15:12,107 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:15:12,107 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:15:12,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-20T21:15:12,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T21:15:12,112 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:15:12,113 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:15:12,114 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:15:12,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T21:15:12,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:12,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137372237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:12,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:12,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137372237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:12,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:12,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:12,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137372240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:12,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:12,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137372241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:12,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:12,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:12,244 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:12,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:12,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137372248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:12,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:12,265 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T21:15:12,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:12,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:12,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:12,266 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:12,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:12,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:12,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T21:15:12,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T21:15:12,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:12,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:12,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:12,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:12,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:12,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=319 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/965f0f041a304aadaf64caf287452253 2024-11-20T21:15:12,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/96d9090a665e47cf95fdc40ab40e1704 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/96d9090a665e47cf95fdc40ab40e1704 2024-11-20T21:15:12,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/96d9090a665e47cf95fdc40ab40e1704, entries=200, sequenceid=319, filesize=14.4 K 2024-11-20T21:15:12,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/64a8353ff03a4a55a52ee390aca73aff as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/64a8353ff03a4a55a52ee390aca73aff 2024-11-20T21:15:12,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/64a8353ff03a4a55a52ee390aca73aff, entries=150, sequenceid=319, filesize=12.0 K 2024-11-20T21:15:12,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/965f0f041a304aadaf64caf287452253 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/965f0f041a304aadaf64caf287452253 2024-11-20T21:15:12,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/965f0f041a304aadaf64caf287452253, entries=150, sequenceid=319, filesize=12.0 K 2024-11-20T21:15:12,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for a5436f89d60238ad7387432d34167717 in 650ms, sequenceid=319, compaction requested=true 2024-11-20T21:15:12,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:12,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:15:12,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:12,491 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:15:12,491 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:15:12,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:12,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:15:12,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:12,492 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:15:12,493 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:15:12,493 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): a5436f89d60238ad7387432d34167717/A is initiating minor compaction (all files) 2024-11-20T21:15:12,493 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5436f89d60238ad7387432d34167717/A in TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:12,493 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/3899816d1ef246288a2896b9be789274, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/744d6a6172814b42b4d91d3588dd2b3d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/b0afda355419416d8d1841a3b19a9b1f, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/96d9090a665e47cf95fdc40ab40e1704] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp, totalSize=51.1 K 2024-11-20T21:15:12,493 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:15:12,494 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): a5436f89d60238ad7387432d34167717/B is initiating minor compaction (all files) 2024-11-20T21:15:12,494 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5436f89d60238ad7387432d34167717/B in TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:12,494 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/f34b88505c30493185e32880d6d72792, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/089221c56d954ca4a4e7602ca7920774, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/0c6c7a3d7c404d629a7faf41cd177f3b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/64a8353ff03a4a55a52ee390aca73aff] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp, totalSize=48.7 K 2024-11-20T21:15:12,494 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3899816d1ef246288a2896b9be789274, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732137310484 2024-11-20T21:15:12,494 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting f34b88505c30493185e32880d6d72792, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732137310484 2024-11-20T21:15:12,494 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 744d6a6172814b42b4d91d3588dd2b3d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732137310630 2024-11-20T21:15:12,495 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 089221c56d954ca4a4e7602ca7920774, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732137310630 2024-11-20T21:15:12,495 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0afda355419416d8d1841a3b19a9b1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732137311427 2024-11-20T21:15:12,495 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c6c7a3d7c404d629a7faf41cd177f3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732137311427 2024-11-20T21:15:12,496 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96d9090a665e47cf95fdc40ab40e1704, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732137311802 2024-11-20T21:15:12,496 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 64a8353ff03a4a55a52ee390aca73aff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732137311802 2024-11-20T21:15:12,510 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5436f89d60238ad7387432d34167717#B#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:12,510 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/0e4300d792824da6acfb9b4480508905 is 50, key is test_row_0/B:col10/1732137311802/Put/seqid=0 2024-11-20T21:15:12,525 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5436f89d60238ad7387432d34167717#A#compaction#280 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:12,526 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/f0b508fbcf5e4760b2afdf8560127254 is 50, key is test_row_0/A:col10/1732137311802/Put/seqid=0 2024-11-20T21:15:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on a5436f89d60238ad7387432d34167717 2024-11-20T21:15:12,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5436f89d60238ad7387432d34167717 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T21:15:12,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=A 2024-11-20T21:15:12,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:12,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=B 2024-11-20T21:15:12,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:12,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=C 2024-11-20T21:15:12,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:12,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742157_1333 (size=13085) 2024-11-20T21:15:12,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/a3ee8916fc23480cad19fc55cc9c1b78 is 50, key is test_row_0/A:col10/1732137312552/Put/seqid=0 2024-11-20T21:15:12,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742156_1332 (size=13085) 2024-11-20T21:15:12,572 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/f0b508fbcf5e4760b2afdf8560127254 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/f0b508fbcf5e4760b2afdf8560127254 2024-11-20T21:15:12,575 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/0e4300d792824da6acfb9b4480508905 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/0e4300d792824da6acfb9b4480508905 2024-11-20T21:15:12,583 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a5436f89d60238ad7387432d34167717/B of a5436f89d60238ad7387432d34167717 into 0e4300d792824da6acfb9b4480508905(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:12,583 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:12,583 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., storeName=a5436f89d60238ad7387432d34167717/B, priority=12, startTime=1732137312491; duration=0sec 2024-11-20T21:15:12,583 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a5436f89d60238ad7387432d34167717/A of a5436f89d60238ad7387432d34167717 into f0b508fbcf5e4760b2afdf8560127254(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:12,583 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:12,583 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:12,583 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., storeName=a5436f89d60238ad7387432d34167717/A, priority=12, startTime=1732137312491; duration=0sec 2024-11-20T21:15:12,583 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:B 2024-11-20T21:15:12,583 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:15:12,584 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:12,584 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:A 2024-11-20T21:15:12,584 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:15:12,584 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): a5436f89d60238ad7387432d34167717/C is initiating minor compaction (all files) 2024-11-20T21:15:12,585 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5436f89d60238ad7387432d34167717/C in TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:12,585 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/590d5a27f472465fab219b854ca82256, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/93134d06f1c144ad9dd978840ab74677, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/8d5a959491ea4b7d96086d01bad28945, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/965f0f041a304aadaf64caf287452253] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp, totalSize=48.7 K 2024-11-20T21:15:12,585 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 590d5a27f472465fab219b854ca82256, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1732137310484 2024-11-20T21:15:12,585 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 93134d06f1c144ad9dd978840ab74677, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1732137310630 2024-11-20T21:15:12,586 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d5a959491ea4b7d96086d01bad28945, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732137311427 2024-11-20T21:15:12,586 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 965f0f041a304aadaf64caf287452253, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732137311802 2024-11-20T21:15:12,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137372577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:12,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137372584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:12,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137372584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:12,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:12,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137372586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:12,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:12,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137372587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:12,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T21:15:12,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:12,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:12,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:12,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:12,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:12,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:12,593 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,600 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:12,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742158_1334 (size=14741) 2024-11-20T21:15:12,616 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5436f89d60238ad7387432d34167717#C#compaction#282 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:12,617 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/22feb88e3a4a49c5828281bca3aad9d3 is 50, key is test_row_0/C:col10/1732137311802/Put/seqid=0 2024-11-20T21:15:12,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742159_1335 (size=13085) 2024-11-20T21:15:12,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:12,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137372697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:12,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:12,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:12,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137372697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:12,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:12,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:12,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137372697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:12,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:12,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137372697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:12,701 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:12,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:12,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:12,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137372707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:12,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,708 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:12,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T21:15:12,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T21:15:12,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:12,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:12,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:12,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:12,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:12,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:12,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:12,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137372907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:12,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:12,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137372907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:12,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:12,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137372907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:12,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:12,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:12,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,911 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:12,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:12,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137372909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:12,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T21:15:12,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:12,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:12,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:12,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:12,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:12,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:12,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,916 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:12,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:12,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137372918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:12,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:12,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:13,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/a3ee8916fc23480cad19fc55cc9c1b78 2024-11-20T21:15:13,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/3f1b017b7ff14015aebb113469e8a549 is 50, key is test_row_0/B:col10/1732137312552/Put/seqid=0 2024-11-20T21:15:13,041 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/22feb88e3a4a49c5828281bca3aad9d3 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/22feb88e3a4a49c5828281bca3aad9d3 2024-11-20T21:15:13,047 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a5436f89d60238ad7387432d34167717/C of a5436f89d60238ad7387432d34167717 into 22feb88e3a4a49c5828281bca3aad9d3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:13,047 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:13,047 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., storeName=a5436f89d60238ad7387432d34167717/C, priority=12, startTime=1732137312492; duration=0sec 2024-11-20T21:15:13,047 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:13,047 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:C 2024-11-20T21:15:13,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T21:15:13,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:13,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:13,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:13,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:13,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:13,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:13,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742160_1336 (size=12301) 2024-11-20T21:15:13,090 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/3f1b017b7ff14015aebb113469e8a549 2024-11-20T21:15:13,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/be29ccbc017d47cfb9482480e49bc778 is 50, key is test_row_0/C:col10/1732137312552/Put/seqid=0 2024-11-20T21:15:13,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742161_1337 (size=12301) 2024-11-20T21:15:13,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/be29ccbc017d47cfb9482480e49bc778 2024-11-20T21:15:13,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/a3ee8916fc23480cad19fc55cc9c1b78 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/a3ee8916fc23480cad19fc55cc9c1b78 2024-11-20T21:15:13,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/a3ee8916fc23480cad19fc55cc9c1b78, entries=200, sequenceid=336, filesize=14.4 K 2024-11-20T21:15:13,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/3f1b017b7ff14015aebb113469e8a549 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/3f1b017b7ff14015aebb113469e8a549 2024-11-20T21:15:13,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/3f1b017b7ff14015aebb113469e8a549, entries=150, sequenceid=336, filesize=12.0 K 2024-11-20T21:15:13,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/be29ccbc017d47cfb9482480e49bc778 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/be29ccbc017d47cfb9482480e49bc778 2024-11-20T21:15:13,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/be29ccbc017d47cfb9482480e49bc778, entries=150, sequenceid=336, filesize=12.0 K 2024-11-20T21:15:13,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for a5436f89d60238ad7387432d34167717 in 633ms, sequenceid=336, compaction requested=false 2024-11-20T21:15:13,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:13,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-20T21:15:13,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:13,233 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing a5436f89d60238ad7387432d34167717 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T21:15:13,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=A 2024-11-20T21:15:13,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:13,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=B 2024-11-20T21:15:13,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:13,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=C 2024-11-20T21:15:13,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:13,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on a5436f89d60238ad7387432d34167717 2024-11-20T21:15:13,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:13,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/3ba9f92da1f84003b03e105609930d33 is 50, key is test_row_0/A:col10/1732137312585/Put/seqid=0 2024-11-20T21:15:13,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T21:15:13,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742162_1338 (size=12301) 2024-11-20T21:15:13,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:13,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137373257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:13,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:13,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:13,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137373265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:13,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:13,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:13,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137373267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:13,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:13,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:13,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137373269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:13,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:13,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137373271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:13,278 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:13,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:13,384 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:13,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137373381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:13,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:13,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137373381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:13,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:13,385 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:13,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:13,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137373387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:13,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:13,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,391 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:13,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137373387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:13,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:13,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137373388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:13,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:13,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:13,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137373589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:13,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,597 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:13,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:13,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:13,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137373598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:13,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137373597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:13,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:13,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:13,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137373601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:13,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:13,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,609 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:13,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:13,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137373607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:13,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:13,675 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/3ba9f92da1f84003b03e105609930d33 2024-11-20T21:15:13,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/7a27248eb7db4862a86aaf3ed54e70bb is 50, key is test_row_0/B:col10/1732137312585/Put/seqid=0 2024-11-20T21:15:13,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742163_1339 (size=12301) 2024-11-20T21:15:13,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:13,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137373908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:13,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,912 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:13,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:13,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137373917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:13,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137373918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:13,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:13,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:13,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137373919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:13,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:13,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:13,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137373919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:13,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:13,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:14,130 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/7a27248eb7db4862a86aaf3ed54e70bb 2024-11-20T21:15:14,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/c26699a95ac7473d9c6b98bcb8fc0048 is 50, key is test_row_0/C:col10/1732137312585/Put/seqid=0 2024-11-20T21:15:14,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742164_1340 (size=12301) 2024-11-20T21:15:14,149 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/c26699a95ac7473d9c6b98bcb8fc0048 2024-11-20T21:15:14,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/3ba9f92da1f84003b03e105609930d33 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/3ba9f92da1f84003b03e105609930d33 2024-11-20T21:15:14,165 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/3ba9f92da1f84003b03e105609930d33, entries=150, sequenceid=356, filesize=12.0 K 2024-11-20T21:15:14,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/7a27248eb7db4862a86aaf3ed54e70bb as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/7a27248eb7db4862a86aaf3ed54e70bb 2024-11-20T21:15:14,171 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/7a27248eb7db4862a86aaf3ed54e70bb, entries=150, sequenceid=356, filesize=12.0 K 2024-11-20T21:15:14,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/c26699a95ac7473d9c6b98bcb8fc0048 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/c26699a95ac7473d9c6b98bcb8fc0048 2024-11-20T21:15:14,181 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/c26699a95ac7473d9c6b98bcb8fc0048, entries=150, sequenceid=356, filesize=12.0 K 2024-11-20T21:15:14,182 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for a5436f89d60238ad7387432d34167717 in 949ms, sequenceid=356, compaction requested=true 2024-11-20T21:15:14,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:14,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:14,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-20T21:15:14,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-20T21:15:14,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-20T21:15:14,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0710 sec 2024-11-20T21:15:14,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.0820 sec 2024-11-20T21:15:14,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-20T21:15:14,261 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:15:14,262 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:15:14,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-20T21:15:14,264 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:15:14,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T21:15:14,266 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:15:14,266 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:15:14,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T21:15:14,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-20T21:15:14,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:14,419 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing a5436f89d60238ad7387432d34167717 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T21:15:14,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:14,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=A 2024-11-20T21:15:14,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on a5436f89d60238ad7387432d34167717 2024-11-20T21:15:14,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:14,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=B 2024-11-20T21:15:14,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:14,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=C 2024-11-20T21:15:14,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:14,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/f45acd17fe054fe18b1bad6fa7247075 is 50, key is test_row_0/A:col10/1732137313264/Put/seqid=0 2024-11-20T21:15:14,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:14,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137374446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:14,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:14,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:14,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:14,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:14,456 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:14,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137374448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:14,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:14,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:14,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:14,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137374446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:14,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:14,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:14,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137374449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:14,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:14,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:14,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:14,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:14,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:14,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:14,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137374452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:14,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:14,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:14,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:14,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742165_1341 (size=14741) 2024-11-20T21:15:14,467 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/f45acd17fe054fe18b1bad6fa7247075 2024-11-20T21:15:14,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/08f65886949b467c840971e94798d35e is 50, key is test_row_0/B:col10/1732137313264/Put/seqid=0 2024-11-20T21:15:14,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742166_1342 (size=12301) 2024-11-20T21:15:14,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:14,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137374567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:14,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:14,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:14,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:14,571 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:14,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137374567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:14,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:14,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:14,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137374567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:14,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:14,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:14,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:14,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:14,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:14,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:14,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137374570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:14,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:14,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137374570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:14,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:14,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:14,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:14,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:14,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:14,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:14,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T21:15:14,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137374777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:14,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:14,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137374777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:14,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:14,782 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:14,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137374777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:14,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:14,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:14,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:14,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:14,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137374787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:14,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:14,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:14,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:14,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:14,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137374788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:14,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:14,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T21:15:14,918 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/08f65886949b467c840971e94798d35e 2024-11-20T21:15:14,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/d5a926179c3045b2bddfae6f3cbfeb10 is 50, key is test_row_0/C:col10/1732137313264/Put/seqid=0 2024-11-20T21:15:15,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742167_1343 (size=12301) 2024-11-20T21:15:15,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:15,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137375086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:15,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:15,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:15,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:15,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:15,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137375087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:15,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:15,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:15,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:15,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:15,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137375087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:15,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137375098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:15,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137375098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:15,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:15,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:15,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:15,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:15,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:15,103 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:15,409 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/d5a926179c3045b2bddfae6f3cbfeb10 2024-11-20T21:15:15,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T21:15:15,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/f45acd17fe054fe18b1bad6fa7247075 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/f45acd17fe054fe18b1bad6fa7247075 2024-11-20T21:15:15,426 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/f45acd17fe054fe18b1bad6fa7247075, entries=200, sequenceid=373, filesize=14.4 K 2024-11-20T21:15:15,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/08f65886949b467c840971e94798d35e as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/08f65886949b467c840971e94798d35e 2024-11-20T21:15:15,432 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/08f65886949b467c840971e94798d35e, entries=150, sequenceid=373, filesize=12.0 K 2024-11-20T21:15:15,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/d5a926179c3045b2bddfae6f3cbfeb10 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/d5a926179c3045b2bddfae6f3cbfeb10 2024-11-20T21:15:15,437 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/d5a926179c3045b2bddfae6f3cbfeb10, entries=150, sequenceid=373, filesize=12.0 K 2024-11-20T21:15:15,438 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for a5436f89d60238ad7387432d34167717 in 1019ms, sequenceid=373, compaction requested=true 2024-11-20T21:15:15,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:15,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:15,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-20T21:15:15,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-20T21:15:15,441 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-20T21:15:15,442 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1740 sec 2024-11-20T21:15:15,454 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.1840 sec 2024-11-20T21:15:15,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on a5436f89d60238ad7387432d34167717 2024-11-20T21:15:15,677 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5436f89d60238ad7387432d34167717 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-20T21:15:15,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=A 2024-11-20T21:15:15,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:15,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=B 2024-11-20T21:15:15,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:15,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=C 2024-11-20T21:15:15,677 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:15,746 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/859d6a667c564ceebb58aa0012e32be3 is 50, key is test_row_0/A:col10/1732137315616/Put/seqid=0 2024-11-20T21:15:15,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:15,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137375728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:15,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:15,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:15,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137375736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:15,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:15,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:15,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137375747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:15,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:15,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:15,766 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:15,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:15,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137375748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:15,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:15,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137375756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:15,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:15,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:15,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:15,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:15,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:15,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:15,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742168_1344 (size=12301) 2024-11-20T21:15:15,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/859d6a667c564ceebb58aa0012e32be3 2024-11-20T21:15:15,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/b57633b3e5a24149a4bd4f7e049a4773 is 50, key is test_row_0/B:col10/1732137315616/Put/seqid=0 2024-11-20T21:15:15,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742169_1345 (size=12301) 2024-11-20T21:15:15,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/b57633b3e5a24149a4bd4f7e049a4773 2024-11-20T21:15:15,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:15,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137375857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:15,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:15,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:15,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:15,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:15,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137375867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:15,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/2efc2c1f3e144b9ab88e715f217d1545 is 50, key is test_row_0/C:col10/1732137315616/Put/seqid=0 2024-11-20T21:15:15,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:15,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:15,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:15,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:15,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137375877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:15,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:15,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:15,897 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:15,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:15,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137375877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:15,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:15,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:15,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137375877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:15,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:15,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:15,904 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:15,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:15,904 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:15,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742170_1346 (size=12301) 2024-11-20T21:15:15,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/2efc2c1f3e144b9ab88e715f217d1545 2024-11-20T21:15:15,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/859d6a667c564ceebb58aa0012e32be3 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/859d6a667c564ceebb58aa0012e32be3 2024-11-20T21:15:15,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/859d6a667c564ceebb58aa0012e32be3, entries=150, sequenceid=391, filesize=12.0 K 2024-11-20T21:15:16,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/b57633b3e5a24149a4bd4f7e049a4773 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/b57633b3e5a24149a4bd4f7e049a4773 2024-11-20T21:15:16,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/b57633b3e5a24149a4bd4f7e049a4773, entries=150, sequenceid=391, filesize=12.0 K 2024-11-20T21:15:16,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/2efc2c1f3e144b9ab88e715f217d1545 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/2efc2c1f3e144b9ab88e715f217d1545 2024-11-20T21:15:16,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/2efc2c1f3e144b9ab88e715f217d1545, entries=150, sequenceid=391, filesize=12.0 K 2024-11-20T21:15:16,023 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for a5436f89d60238ad7387432d34167717 in 347ms, sequenceid=391, compaction requested=true 2024-11-20T21:15:16,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:16,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:15:16,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:16,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:15:16,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T21:15:16,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:15:16,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-20T21:15:16,024 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T21:15:16,024 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T21:15:16,041 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 67169 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T21:15:16,041 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): a5436f89d60238ad7387432d34167717/A is initiating minor compaction (all files) 2024-11-20T21:15:16,042 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5436f89d60238ad7387432d34167717/A in TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:16,042 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/f0b508fbcf5e4760b2afdf8560127254, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/a3ee8916fc23480cad19fc55cc9c1b78, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/3ba9f92da1f84003b03e105609930d33, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/f45acd17fe054fe18b1bad6fa7247075, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/859d6a667c564ceebb58aa0012e32be3] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp, totalSize=65.6 K 2024-11-20T21:15:16,042 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62289 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T21:15:16,042 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): a5436f89d60238ad7387432d34167717/C is initiating minor compaction (all files) 2024-11-20T21:15:16,042 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5436f89d60238ad7387432d34167717/C in TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:16,042 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/22feb88e3a4a49c5828281bca3aad9d3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/be29ccbc017d47cfb9482480e49bc778, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/c26699a95ac7473d9c6b98bcb8fc0048, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/d5a926179c3045b2bddfae6f3cbfeb10, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/2efc2c1f3e144b9ab88e715f217d1545] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp, totalSize=60.8 K 2024-11-20T21:15:16,045 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 22feb88e3a4a49c5828281bca3aad9d3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732137311802 2024-11-20T21:15:16,045 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0b508fbcf5e4760b2afdf8560127254, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732137311802 2024-11-20T21:15:16,046 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting be29ccbc017d47cfb9482480e49bc778, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732137311887 2024-11-20T21:15:16,046 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3ee8916fc23480cad19fc55cc9c1b78, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732137311886 2024-11-20T21:15:16,046 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting c26699a95ac7473d9c6b98bcb8fc0048, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732137312581 2024-11-20T21:15:16,046 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ba9f92da1f84003b03e105609930d33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732137312581 2024-11-20T21:15:16,047 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting d5a926179c3045b2bddfae6f3cbfeb10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732137313254 2024-11-20T21:15:16,047 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f45acd17fe054fe18b1bad6fa7247075, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732137313254 2024-11-20T21:15:16,047 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 2efc2c1f3e144b9ab88e715f217d1545, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732137314448 2024-11-20T21:15:16,054 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 859d6a667c564ceebb58aa0012e32be3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732137314448 2024-11-20T21:15:16,076 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5436f89d60238ad7387432d34167717#A#compaction#294 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:16,076 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/9ea6561546de44aab4eb36daab164d82 is 50, key is test_row_0/A:col10/1732137315616/Put/seqid=0 2024-11-20T21:15:16,090 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5436f89d60238ad7387432d34167717#C#compaction#295 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:16,091 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/77abb8997ce94ce1aff5243d6f742d97 is 50, key is test_row_0/C:col10/1732137315616/Put/seqid=0 2024-11-20T21:15:16,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on a5436f89d60238ad7387432d34167717 2024-11-20T21:15:16,095 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5436f89d60238ad7387432d34167717 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T21:15:16,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=A 2024-11-20T21:15:16,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:16,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=B 2024-11-20T21:15:16,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:16,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=C 2024-11-20T21:15:16,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:16,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/b07f0c62914742f791cb7fe2c3a92eb2 is 50, key is test_row_0/A:col10/1732137316093/Put/seqid=0 2024-11-20T21:15:16,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742172_1348 (size=13255) 2024-11-20T21:15:16,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742171_1347 (size=13255) 2024-11-20T21:15:16,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:16,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137376149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:16,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:16,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137376152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:16,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:16,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137376153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:16,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:16,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:16,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137376163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:16,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:16,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742173_1349 (size=17181) 2024-11-20T21:15:16,182 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/9ea6561546de44aab4eb36daab164d82 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/9ea6561546de44aab4eb36daab164d82 2024-11-20T21:15:16,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/b07f0c62914742f791cb7fe2c3a92eb2 2024-11-20T21:15:16,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:16,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137376171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:16,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:16,201 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in a5436f89d60238ad7387432d34167717/A of a5436f89d60238ad7387432d34167717 into 9ea6561546de44aab4eb36daab164d82(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:16,201 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:16,201 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., storeName=a5436f89d60238ad7387432d34167717/A, priority=11, startTime=1732137316024; duration=0sec 2024-11-20T21:15:16,201 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:16,201 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:A 2024-11-20T21:15:16,201 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T21:15:16,203 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62289 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T21:15:16,203 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): a5436f89d60238ad7387432d34167717/B is initiating minor compaction (all files) 2024-11-20T21:15:16,203 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5436f89d60238ad7387432d34167717/B in TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:16,203 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/0e4300d792824da6acfb9b4480508905, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/3f1b017b7ff14015aebb113469e8a549, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/7a27248eb7db4862a86aaf3ed54e70bb, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/08f65886949b467c840971e94798d35e, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/b57633b3e5a24149a4bd4f7e049a4773] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp, totalSize=60.8 K 2024-11-20T21:15:16,204 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e4300d792824da6acfb9b4480508905, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=319, earliestPutTs=1732137311802 2024-11-20T21:15:16,204 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f1b017b7ff14015aebb113469e8a549, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1732137311887 2024-11-20T21:15:16,204 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a27248eb7db4862a86aaf3ed54e70bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732137312581 2024-11-20T21:15:16,205 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08f65886949b467c840971e94798d35e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1732137313254 2024-11-20T21:15:16,205 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting b57633b3e5a24149a4bd4f7e049a4773, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732137314448 2024-11-20T21:15:16,210 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/45d6991ba512433baff63ce806dcaf89 is 50, key is test_row_0/B:col10/1732137316093/Put/seqid=0 2024-11-20T21:15:16,256 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5436f89d60238ad7387432d34167717#B#compaction#298 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:16,257 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/904c6362a9fb4123a3de2213ea16d06d is 50, key is test_row_0/B:col10/1732137315616/Put/seqid=0 2024-11-20T21:15:16,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742174_1350 (size=12301) 2024-11-20T21:15:16,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/45d6991ba512433baff63ce806dcaf89 2024-11-20T21:15:16,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742175_1351 (size=13255) 2024-11-20T21:15:16,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:16,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137376277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:16,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,285 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:16,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/58e6c37802074388850e8a82ae5a9578 is 50, key is test_row_0/C:col10/1732137316093/Put/seqid=0 2024-11-20T21:15:16,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137376287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:16,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:16,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:16,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137376287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137376287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:16,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137376307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:16,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742176_1352 (size=12301) 2024-11-20T21:15:16,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=409 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/58e6c37802074388850e8a82ae5a9578 2024-11-20T21:15:16,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/b07f0c62914742f791cb7fe2c3a92eb2 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/b07f0c62914742f791cb7fe2c3a92eb2 2024-11-20T21:15:16,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/b07f0c62914742f791cb7fe2c3a92eb2, entries=250, sequenceid=409, filesize=16.8 K 2024-11-20T21:15:16,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/45d6991ba512433baff63ce806dcaf89 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/45d6991ba512433baff63ce806dcaf89 2024-11-20T21:15:16,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/45d6991ba512433baff63ce806dcaf89, entries=150, sequenceid=409, filesize=12.0 K 2024-11-20T21:15:16,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/58e6c37802074388850e8a82ae5a9578 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/58e6c37802074388850e8a82ae5a9578 2024-11-20T21:15:16,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/58e6c37802074388850e8a82ae5a9578, entries=150, sequenceid=409, filesize=12.0 K 2024-11-20T21:15:16,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for a5436f89d60238ad7387432d34167717 in 256ms, sequenceid=409, compaction requested=false 2024-11-20T21:15:16,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:16,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-20T21:15:16,427 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:15:16,428 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:15:16,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-20T21:15:16,430 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:15:16,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T21:15:16,432 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:15:16,432 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:15:16,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5436f89d60238ad7387432d34167717 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T21:15:16,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on a5436f89d60238ad7387432d34167717 2024-11-20T21:15:16,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=A 2024-11-20T21:15:16,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:16,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=B 2024-11-20T21:15:16,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:16,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=C 2024-11-20T21:15:16,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:16,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/aad7e7ace5414dc98e73389652bf76d2 is 50, key is test_row_0/A:col10/1732137316149/Put/seqid=0 2024-11-20T21:15:16,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:16,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137376530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:16,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T21:15:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:16,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:16,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137376531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:16,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:16,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:16,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137376533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:16,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:16,548 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/77abb8997ce94ce1aff5243d6f742d97 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/77abb8997ce94ce1aff5243d6f742d97 2024-11-20T21:15:16,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:16,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137376535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:16,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:16,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137376537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:16,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:16,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:16,556 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in a5436f89d60238ad7387432d34167717/C of a5436f89d60238ad7387432d34167717 into 77abb8997ce94ce1aff5243d6f742d97(size=12.9 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:16,556 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:16,556 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., storeName=a5436f89d60238ad7387432d34167717/C, priority=11, startTime=1732137316024; duration=0sec 2024-11-20T21:15:16,556 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:16,556 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:C 2024-11-20T21:15:16,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742177_1353 (size=12301) 2024-11-20T21:15:16,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/aad7e7ace5414dc98e73389652bf76d2 2024-11-20T21:15:16,583 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/77509657f9d146978ea66dc34cde3c03 is 50, key is test_row_0/B:col10/1732137316149/Put/seqid=0 2024-11-20T21:15:16,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T21:15:16,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:16,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:16,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:16,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:16,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:16,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:16,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742178_1354 (size=12301) 2024-11-20T21:15:16,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/77509657f9d146978ea66dc34cde3c03 2024-11-20T21:15:16,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:16,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137376647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:16,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:16,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:16,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137376647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:16,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:16,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:16,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137376647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:16,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:16,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:16,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137376657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:16,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:16,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137376657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:16,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:16,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,662 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:16,688 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/0a22a2d6f44349c489350f9e5219c008 is 50, key is test_row_0/C:col10/1732137316149/Put/seqid=0 2024-11-20T21:15:16,706 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/904c6362a9fb4123a3de2213ea16d06d as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/904c6362a9fb4123a3de2213ea16d06d 2024-11-20T21:15:16,712 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in a5436f89d60238ad7387432d34167717/B of a5436f89d60238ad7387432d34167717 into 904c6362a9fb4123a3de2213ea16d06d(size=12.9 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:16,712 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:16,712 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., storeName=a5436f89d60238ad7387432d34167717/B, priority=11, startTime=1732137316024; duration=0sec 2024-11-20T21:15:16,712 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:16,712 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:B 2024-11-20T21:15:16,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742179_1355 (size=12301) 2024-11-20T21:15:16,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=427 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/0a22a2d6f44349c489350f9e5219c008 2024-11-20T21:15:16,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T21:15:16,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:16,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:16,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:16,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:16,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:16,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:16,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T21:15:16,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/aad7e7ace5414dc98e73389652bf76d2 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/aad7e7ace5414dc98e73389652bf76d2 2024-11-20T21:15:16,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/aad7e7ace5414dc98e73389652bf76d2, entries=150, sequenceid=427, filesize=12.0 K 2024-11-20T21:15:16,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/77509657f9d146978ea66dc34cde3c03 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/77509657f9d146978ea66dc34cde3c03 2024-11-20T21:15:16,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/77509657f9d146978ea66dc34cde3c03, entries=150, sequenceid=427, filesize=12.0 K 2024-11-20T21:15:16,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/0a22a2d6f44349c489350f9e5219c008 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/0a22a2d6f44349c489350f9e5219c008 2024-11-20T21:15:16,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/0a22a2d6f44349c489350f9e5219c008, entries=150, sequenceid=427, filesize=12.0 K 2024-11-20T21:15:16,778 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for a5436f89d60238ad7387432d34167717 in 276ms, sequenceid=427, compaction requested=true 2024-11-20T21:15:16,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:16,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:15:16,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:16,778 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:16,778 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:16,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:15:16,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:16,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:15:16,778 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:16,780 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:16,780 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42737 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:16,780 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): a5436f89d60238ad7387432d34167717/B is initiating minor compaction (all files) 2024-11-20T21:15:16,780 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): a5436f89d60238ad7387432d34167717/A is initiating minor compaction (all files) 2024-11-20T21:15:16,780 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5436f89d60238ad7387432d34167717/B in TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:16,780 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5436f89d60238ad7387432d34167717/A in TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:16,780 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/9ea6561546de44aab4eb36daab164d82, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/b07f0c62914742f791cb7fe2c3a92eb2, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/aad7e7ace5414dc98e73389652bf76d2] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp, totalSize=41.7 K 2024-11-20T21:15:16,780 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/904c6362a9fb4123a3de2213ea16d06d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/45d6991ba512433baff63ce806dcaf89, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/77509657f9d146978ea66dc34cde3c03] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp, totalSize=37.0 K 2024-11-20T21:15:16,781 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 904c6362a9fb4123a3de2213ea16d06d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732137314448 2024-11-20T21:15:16,781 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ea6561546de44aab4eb36daab164d82, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732137314448 2024-11-20T21:15:16,782 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 45d6991ba512433baff63ce806dcaf89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732137315743 2024-11-20T21:15:16,782 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting b07f0c62914742f791cb7fe2c3a92eb2, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732137315743 2024-11-20T21:15:16,782 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting aad7e7ace5414dc98e73389652bf76d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1732137316149 2024-11-20T21:15:16,782 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 77509657f9d146978ea66dc34cde3c03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1732137316149 2024-11-20T21:15:16,793 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5436f89d60238ad7387432d34167717#A#compaction#303 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:16,794 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/107e7d97a45f4215bdba5cfbd7c08c01 is 50, key is test_row_0/A:col10/1732137316149/Put/seqid=0 2024-11-20T21:15:16,797 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5436f89d60238ad7387432d34167717#B#compaction#304 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:16,797 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/6cd3e609eeab49d69d691b4dcfa88083 is 50, key is test_row_0/B:col10/1732137316149/Put/seqid=0 2024-11-20T21:15:16,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742180_1356 (size=13357) 2024-11-20T21:15:16,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742181_1357 (size=13357) 2024-11-20T21:15:16,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on a5436f89d60238ad7387432d34167717 2024-11-20T21:15:16,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5436f89d60238ad7387432d34167717 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T21:15:16,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=A 2024-11-20T21:15:16,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:16,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=B 2024-11-20T21:15:16,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:16,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=C 2024-11-20T21:15:16,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:16,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/2ddd671d940648559c561f5fae1b8896 is 50, key is test_row_0/A:col10/1732137316868/Put/seqid=0 2024-11-20T21:15:16,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T21:15:16,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:16,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:16,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:16,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:16,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:16,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:16,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742182_1358 (size=12301) 2024-11-20T21:15:16,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137376946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:16,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:16,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:16,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137376946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:16,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137376949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,956 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:16,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:16,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137376952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:16,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:16,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:16,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137376955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:16,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:16,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:17,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T21:15:17,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:17,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:17,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:17,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:17,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:17,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:17,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T21:15:17,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:17,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137377057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:17,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:17,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:17,061 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:17,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137377067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:17,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:17,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:17,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137377067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:17,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:17,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:17,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:17,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:17,070 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:17,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137377067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:17,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:17,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:17,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:17,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:17,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137377067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:17,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:17,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:17,071 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:17,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T21:15:17,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:17,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:17,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:17,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:17,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:17,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:17,259 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/107e7d97a45f4215bdba5cfbd7c08c01 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/107e7d97a45f4215bdba5cfbd7c08c01 2024-11-20T21:15:17,262 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/6cd3e609eeab49d69d691b4dcfa88083 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/6cd3e609eeab49d69d691b4dcfa88083 2024-11-20T21:15:17,264 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5436f89d60238ad7387432d34167717/A of a5436f89d60238ad7387432d34167717 into 107e7d97a45f4215bdba5cfbd7c08c01(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:17,264 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:17,264 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., storeName=a5436f89d60238ad7387432d34167717/A, priority=13, startTime=1732137316778; duration=0sec 2024-11-20T21:15:17,264 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:17,264 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:A 2024-11-20T21:15:17,264 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:17,265 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:17,265 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): a5436f89d60238ad7387432d34167717/C is initiating minor compaction (all files) 2024-11-20T21:15:17,265 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5436f89d60238ad7387432d34167717/C in TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:17,265 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/77abb8997ce94ce1aff5243d6f742d97, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/58e6c37802074388850e8a82ae5a9578, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/0a22a2d6f44349c489350f9e5219c008] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp, totalSize=37.0 K 2024-11-20T21:15:17,266 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77abb8997ce94ce1aff5243d6f742d97, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1732137314448 2024-11-20T21:15:17,266 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58e6c37802074388850e8a82ae5a9578, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=409, earliestPutTs=1732137315743 2024-11-20T21:15:17,266 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5436f89d60238ad7387432d34167717/B of a5436f89d60238ad7387432d34167717 into 6cd3e609eeab49d69d691b4dcfa88083(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:17,266 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:17,266 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., storeName=a5436f89d60238ad7387432d34167717/B, priority=13, startTime=1732137316778; duration=0sec 2024-11-20T21:15:17,266 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:17,266 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:B 2024-11-20T21:15:17,266 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a22a2d6f44349c489350f9e5219c008, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1732137316149 2024-11-20T21:15:17,272 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5436f89d60238ad7387432d34167717#C#compaction#306 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:17,273 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/adc246b373c940ec88e376ab4997bbf0 is 50, key is test_row_0/C:col10/1732137316149/Put/seqid=0 2024-11-20T21:15:17,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:17,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137377267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:17,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:17,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:17,274 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:17,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742183_1359 (size=13357) 2024-11-20T21:15:17,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:17,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137377277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:17,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:17,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:17,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:17,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:17,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137377277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:17,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:17,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:17,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:17,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:17,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137377277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:17,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:17,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:17,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:17,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:17,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137377277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:17,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:17,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:17,283 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:17,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/2ddd671d940648559c561f5fae1b8896 2024-11-20T21:15:17,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/f94bc79718da4237a82f5b03f0d15f19 is 50, key is test_row_0/B:col10/1732137316868/Put/seqid=0 2024-11-20T21:15:17,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742184_1360 (size=12301) 2024-11-20T21:15:17,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T21:15:17,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:17,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:17,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:17,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:17,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:17,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:17,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T21:15:17,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:17,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:17,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:17,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:17,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:17,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:17,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T21:15:17,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:17,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137377587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:17,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:17,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:17,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137377587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:17,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:17,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:17,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:17,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:17,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:17,590 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:17,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137377587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:17,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:17,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:17,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:17,591 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:17,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137377588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:17,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:17,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:17,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:17,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:17,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137377588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:17,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:17,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:17,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:17,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T21:15:17,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:17,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:17,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:17,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:17,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:17,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:17,685 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/adc246b373c940ec88e376ab4997bbf0 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/adc246b373c940ec88e376ab4997bbf0 2024-11-20T21:15:17,690 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5436f89d60238ad7387432d34167717/C of a5436f89d60238ad7387432d34167717 into adc246b373c940ec88e376ab4997bbf0(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:17,690 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:17,690 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., storeName=a5436f89d60238ad7387432d34167717/C, priority=13, startTime=1732137316778; duration=0sec 2024-11-20T21:15:17,690 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:17,690 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:C 2024-11-20T21:15:17,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/f94bc79718da4237a82f5b03f0d15f19 2024-11-20T21:15:17,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/b1aaab2924494e52a30298b1dbb42a32 is 50, key is test_row_0/C:col10/1732137316868/Put/seqid=0 2024-11-20T21:15:17,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742185_1361 (size=12301) 2024-11-20T21:15:17,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T21:15:17,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:17,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:17,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:17,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:17,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:17,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:17,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T21:15:17,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:17,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:17,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:17,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:17,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:17,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:18,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:18,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137378097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:18,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:18,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137378097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:18,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:18,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:18,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:18,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137378097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:18,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:18,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:18,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:18,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:18,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:18,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:18,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137378098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:18,103 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:18,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137378098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:18,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:18,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:18,104 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:18,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:18,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:18,104 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:18,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T21:15:18,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:18,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:18,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:18,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:18,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:18,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:18,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=446 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/b1aaab2924494e52a30298b1dbb42a32 2024-11-20T21:15:18,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/2ddd671d940648559c561f5fae1b8896 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/2ddd671d940648559c561f5fae1b8896 2024-11-20T21:15:18,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/2ddd671d940648559c561f5fae1b8896, entries=150, sequenceid=446, filesize=12.0 K 2024-11-20T21:15:18,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/f94bc79718da4237a82f5b03f0d15f19 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/f94bc79718da4237a82f5b03f0d15f19 2024-11-20T21:15:18,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/f94bc79718da4237a82f5b03f0d15f19, entries=150, sequenceid=446, filesize=12.0 K 2024-11-20T21:15:18,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/b1aaab2924494e52a30298b1dbb42a32 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/b1aaab2924494e52a30298b1dbb42a32 2024-11-20T21:15:18,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/b1aaab2924494e52a30298b1dbb42a32, entries=150, sequenceid=446, filesize=12.0 K 2024-11-20T21:15:18,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for a5436f89d60238ad7387432d34167717 in 1298ms, sequenceid=446, compaction requested=false 2024-11-20T21:15:18,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:18,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-20T21:15:18,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:18,280 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing a5436f89d60238ad7387432d34167717 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-20T21:15:18,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=A 2024-11-20T21:15:18,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:18,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=B 2024-11-20T21:15:18,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:18,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=C 2024-11-20T21:15:18,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:18,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/e77dc428c4024908be1835de124fea7d is 50, key is test_row_0/A:col10/1732137316941/Put/seqid=0 2024-11-20T21:15:18,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742186_1362 (size=12301) 2024-11-20T21:15:18,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T21:15:18,755 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/e77dc428c4024908be1835de124fea7d 2024-11-20T21:15:18,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/2974c913612d4fdbb67178cb39645993 is 50, key is test_row_0/B:col10/1732137316941/Put/seqid=0 2024-11-20T21:15:18,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742187_1363 (size=12301) 2024-11-20T21:15:18,822 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/2974c913612d4fdbb67178cb39645993 2024-11-20T21:15:18,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/a49a9b19cbf848ee802090f6fa025e73 is 50, key is test_row_0/C:col10/1732137316941/Put/seqid=0 2024-11-20T21:15:18,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742188_1364 (size=12301) 2024-11-20T21:15:18,878 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/a49a9b19cbf848ee802090f6fa025e73 2024-11-20T21:15:18,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/e77dc428c4024908be1835de124fea7d as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/e77dc428c4024908be1835de124fea7d 2024-11-20T21:15:18,897 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/e77dc428c4024908be1835de124fea7d, entries=150, sequenceid=464, filesize=12.0 K 2024-11-20T21:15:18,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/2974c913612d4fdbb67178cb39645993 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/2974c913612d4fdbb67178cb39645993 2024-11-20T21:15:18,906 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/2974c913612d4fdbb67178cb39645993, entries=150, sequenceid=464, filesize=12.0 K 2024-11-20T21:15:18,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/a49a9b19cbf848ee802090f6fa025e73 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/a49a9b19cbf848ee802090f6fa025e73 2024-11-20T21:15:18,916 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/a49a9b19cbf848ee802090f6fa025e73, entries=150, sequenceid=464, filesize=12.0 K 2024-11-20T21:15:18,919 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for a5436f89d60238ad7387432d34167717 in 638ms, sequenceid=464, compaction requested=true 2024-11-20T21:15:18,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:18,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:18,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-20T21:15:18,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-20T21:15:18,928 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-20T21:15:18,928 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4930 sec 2024-11-20T21:15:18,931 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.5010 sec 2024-11-20T21:15:19,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on a5436f89d60238ad7387432d34167717 2024-11-20T21:15:19,138 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5436f89d60238ad7387432d34167717 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T21:15:19,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=A 2024-11-20T21:15:19,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:19,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=B 2024-11-20T21:15:19,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:19,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=C 2024-11-20T21:15:19,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:19,159 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/aad404b478804682a53c74a9503a5c64 is 50, key is test_row_0/A:col10/1732137319137/Put/seqid=0 2024-11-20T21:15:19,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732137319182, max timestamp=1732137319189Num uniques 859; 2024-11-20T21:15:19,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T21:15:19,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742189_1365 (size=12301) 2024-11-20T21:15:19,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:19,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137379197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:19,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:19,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:19,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137379197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:19,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:19,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:19,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137379209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:19,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:19,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137379209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:19,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:19,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137379213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:19,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:19,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:19,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,229 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:19,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:19,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137379317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:19,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:19,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137379318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:19,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:19,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,322 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:19,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:19,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137379327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:19,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137379337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:19,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137379337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:19,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,344 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:19,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:19,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:19,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137379529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:19,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:19,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:19,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137379533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:19,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:19,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:19,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137379545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:19,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:19,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:19,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137379548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:19,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:19,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:19,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137379548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:19,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,553 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:19,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/aad404b478804682a53c74a9503a5c64 2024-11-20T21:15:19,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/e42c90442f5c47168f9df624152c0f34 is 50, key is test_row_0/B:col10/1732137319137/Put/seqid=0 2024-11-20T21:15:19,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742190_1366 (size=12301) 2024-11-20T21:15:19,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/e42c90442f5c47168f9df624152c0f34 2024-11-20T21:15:19,645 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/61642adcb0e94096ba9056110f597c84 is 50, key is test_row_0/C:col10/1732137319137/Put/seqid=0 2024-11-20T21:15:19,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742191_1367 (size=12301) 2024-11-20T21:15:19,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:19,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137379847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:19,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,852 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:19,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:19,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137379850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:19,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:19,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137379858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:19,864 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,865 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:19,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:19,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137379858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:19,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:19,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:19,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137379858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:19,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,868 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:19,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:19,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:20,061 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=476 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/61642adcb0e94096ba9056110f597c84 2024-11-20T21:15:20,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/aad404b478804682a53c74a9503a5c64 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/aad404b478804682a53c74a9503a5c64 2024-11-20T21:15:20,081 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/aad404b478804682a53c74a9503a5c64, entries=150, sequenceid=476, filesize=12.0 K 2024-11-20T21:15:20,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/e42c90442f5c47168f9df624152c0f34 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/e42c90442f5c47168f9df624152c0f34 2024-11-20T21:15:20,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/e42c90442f5c47168f9df624152c0f34, entries=150, sequenceid=476, filesize=12.0 K 2024-11-20T21:15:20,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/61642adcb0e94096ba9056110f597c84 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/61642adcb0e94096ba9056110f597c84 2024-11-20T21:15:20,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/61642adcb0e94096ba9056110f597c84, entries=150, sequenceid=476, filesize=12.0 K 2024-11-20T21:15:20,098 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=118.53 KB/121370 for a5436f89d60238ad7387432d34167717 in 960ms, sequenceid=476, compaction requested=true 2024-11-20T21:15:20,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:20,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:15:20,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:20,098 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:15:20,098 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:15:20,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:15:20,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:20,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:15:20,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:20,100 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50260 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:15:20,100 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): a5436f89d60238ad7387432d34167717/A is initiating minor compaction (all files) 2024-11-20T21:15:20,100 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5436f89d60238ad7387432d34167717/A in TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:20,100 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/107e7d97a45f4215bdba5cfbd7c08c01, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/2ddd671d940648559c561f5fae1b8896, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/e77dc428c4024908be1835de124fea7d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/aad404b478804682a53c74a9503a5c64] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp, totalSize=49.1 K 2024-11-20T21:15:20,101 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50260 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:15:20,101 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): a5436f89d60238ad7387432d34167717/B is initiating minor compaction (all files) 2024-11-20T21:15:20,101 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5436f89d60238ad7387432d34167717/B in TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:20,101 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/6cd3e609eeab49d69d691b4dcfa88083, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/f94bc79718da4237a82f5b03f0d15f19, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/2974c913612d4fdbb67178cb39645993, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/e42c90442f5c47168f9df624152c0f34] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp, totalSize=49.1 K 2024-11-20T21:15:20,104 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 107e7d97a45f4215bdba5cfbd7c08c01, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1732137316149 2024-11-20T21:15:20,109 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cd3e609eeab49d69d691b4dcfa88083, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1732137316149 2024-11-20T21:15:20,110 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ddd671d940648559c561f5fae1b8896, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1732137316533 2024-11-20T21:15:20,110 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting f94bc79718da4237a82f5b03f0d15f19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1732137316533 2024-11-20T21:15:20,110 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting e77dc428c4024908be1835de124fea7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1732137316907 2024-11-20T21:15:20,111 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 2974c913612d4fdbb67178cb39645993, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1732137316907 2024-11-20T21:15:20,111 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting aad404b478804682a53c74a9503a5c64, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1732137319135 2024-11-20T21:15:20,111 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting e42c90442f5c47168f9df624152c0f34, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1732137319135 2024-11-20T21:15:20,160 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5436f89d60238ad7387432d34167717#B#compaction#315 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:20,160 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/9fa4727c2e174ef88a5f1994be54ffcd is 50, key is test_row_0/B:col10/1732137319137/Put/seqid=0 2024-11-20T21:15:20,169 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5436f89d60238ad7387432d34167717#A#compaction#316 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:20,170 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/3792b5a0aebe443c84cffc0ac4588e44 is 50, key is test_row_0/A:col10/1732137319137/Put/seqid=0 2024-11-20T21:15:20,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742192_1368 (size=13493) 2024-11-20T21:15:20,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742193_1369 (size=13493) 2024-11-20T21:15:20,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on a5436f89d60238ad7387432d34167717 2024-11-20T21:15:20,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5436f89d60238ad7387432d34167717 3/3 column families, dataSize=125.23 KB heapSize=366.80 KB 2024-11-20T21:15:20,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=A 2024-11-20T21:15:20,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:20,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=B 2024-11-20T21:15:20,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:20,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=C 2024-11-20T21:15:20,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:20,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/f4da54c913114f6f9e8960e72ac8ffd1 is 50, key is test_row_0/A:col10/1732137319189/Put/seqid=0 2024-11-20T21:15:20,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:20,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137380385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:20,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:20,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742194_1370 (size=12301) 2024-11-20T21:15:20,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/f4da54c913114f6f9e8960e72ac8ffd1 2024-11-20T21:15:20,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:20,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137380394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137380392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:20,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:20,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137380394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:20,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,403 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:20,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:20,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137380394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:20,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:20,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:20,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:20,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/3085138e0aa743f689069d684d854ed5 is 50, key is test_row_0/B:col10/1732137319189/Put/seqid=0 2024-11-20T21:15:20,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742195_1371 (size=12301) 2024-11-20T21:15:20,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/3085138e0aa743f689069d684d854ed5 2024-11-20T21:15:20,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/f9fd07b8dfed4ac7a368f0ca9a24a075 is 50, key is test_row_0/C:col10/1732137319189/Put/seqid=0 2024-11-20T21:15:20,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:20,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137380496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:20,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:20,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742196_1372 (size=12301) 2024-11-20T21:15:20,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/f9fd07b8dfed4ac7a368f0ca9a24a075 2024-11-20T21:15:20,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:20,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137380507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:20,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137380507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:20,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/f4da54c913114f6f9e8960e72ac8ffd1 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/f4da54c913114f6f9e8960e72ac8ffd1 2024-11-20T21:15:20,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/f4da54c913114f6f9e8960e72ac8ffd1, entries=150, sequenceid=502, filesize=12.0 K 2024-11-20T21:15:20,520 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/3085138e0aa743f689069d684d854ed5 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/3085138e0aa743f689069d684d854ed5 2024-11-20T21:15:20,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137380517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:20,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:20,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137380518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:20,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,524 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:20,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/3085138e0aa743f689069d684d854ed5, entries=150, sequenceid=502, filesize=12.0 K 2024-11-20T21:15:20,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/f9fd07b8dfed4ac7a368f0ca9a24a075 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/f9fd07b8dfed4ac7a368f0ca9a24a075 2024-11-20T21:15:20,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/f9fd07b8dfed4ac7a368f0ca9a24a075, entries=150, sequenceid=502, filesize=12.0 K 2024-11-20T21:15:20,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~125.23 KB/128240, heapSize ~366.75 KB/375552, currentSize=53.67 KB/54960 for a5436f89d60238ad7387432d34167717 in 158ms, sequenceid=502, compaction requested=true 2024-11-20T21:15:20,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:20,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:A, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:15:20,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T21:15:20,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:B, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:15:20,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-20T21:15:20,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:15:20,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-20T21:15:20,581 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/9fa4727c2e174ef88a5f1994be54ffcd as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/9fa4727c2e174ef88a5f1994be54ffcd 2024-11-20T21:15:20,586 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a5436f89d60238ad7387432d34167717/B of a5436f89d60238ad7387432d34167717 into 9fa4727c2e174ef88a5f1994be54ffcd(size=13.2 K), total size for store is 25.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:20,586 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:20,586 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., storeName=a5436f89d60238ad7387432d34167717/B, priority=12, startTime=1732137320098; duration=0sec 2024-11-20T21:15:20,586 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-20T21:15:20,586 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:B 2024-11-20T21:15:20,586 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 4 compacting, 1 eligible, 16 blocking 2024-11-20T21:15:20,586 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T21:15:20,586 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T21:15:20,586 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. because compaction request was cancelled 2024-11-20T21:15:20,586 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:A 2024-11-20T21:15:20,587 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-20T21:15:20,588 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62561 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-20T21:15:20,588 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): a5436f89d60238ad7387432d34167717/C is initiating minor compaction (all files) 2024-11-20T21:15:20,588 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5436f89d60238ad7387432d34167717/C in TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:20,588 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/adc246b373c940ec88e376ab4997bbf0, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/b1aaab2924494e52a30298b1dbb42a32, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/a49a9b19cbf848ee802090f6fa025e73, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/61642adcb0e94096ba9056110f597c84, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/f9fd07b8dfed4ac7a368f0ca9a24a075] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp, totalSize=61.1 K 2024-11-20T21:15:20,588 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting adc246b373c940ec88e376ab4997bbf0, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=427, earliestPutTs=1732137316149 2024-11-20T21:15:20,589 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting b1aaab2924494e52a30298b1dbb42a32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=446, earliestPutTs=1732137316533 2024-11-20T21:15:20,589 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting a49a9b19cbf848ee802090f6fa025e73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1732137316907 2024-11-20T21:15:20,589 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 61642adcb0e94096ba9056110f597c84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1732137319135 2024-11-20T21:15:20,590 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting f9fd07b8dfed4ac7a368f0ca9a24a075, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1732137319189 2024-11-20T21:15:20,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-20T21:15:20,597 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:15:20,598 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:15:20,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-20T21:15:20,599 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5436f89d60238ad7387432d34167717#C#compaction#320 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:20,599 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/a4854849cd634dd2b34f5b268d21cf0a is 50, key is test_row_0/C:col10/1732137319189/Put/seqid=0 2024-11-20T21:15:20,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T21:15:20,600 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:15:20,601 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:15:20,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:15:20,609 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/3792b5a0aebe443c84cffc0ac4588e44 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/3792b5a0aebe443c84cffc0ac4588e44 2024-11-20T21:15:20,614 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a5436f89d60238ad7387432d34167717/A of a5436f89d60238ad7387432d34167717 into 3792b5a0aebe443c84cffc0ac4588e44(size=13.2 K), total size for store is 25.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:20,614 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:20,614 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., storeName=a5436f89d60238ad7387432d34167717/A, priority=12, startTime=1732137320098; duration=0sec 2024-11-20T21:15:20,615 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T21:15:20,615 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:A 2024-11-20T21:15:20,615 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:B 2024-11-20T21:15:20,615 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 5 compacting, 0 eligible, 16 blocking 2024-11-20T21:15:20,615 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T21:15:20,615 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T21:15:20,615 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. because compaction request was cancelled 2024-11-20T21:15:20,615 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:C 2024-11-20T21:15:20,615 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-20T21:15:20,616 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T21:15:20,616 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T21:15:20,616 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. because compaction request was cancelled 2024-11-20T21:15:20,616 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:B 2024-11-20T21:15:20,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742197_1373 (size=13527) 2024-11-20T21:15:20,647 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/a4854849cd634dd2b34f5b268d21cf0a as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/a4854849cd634dd2b34f5b268d21cf0a 2024-11-20T21:15:20,657 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in a5436f89d60238ad7387432d34167717/C of a5436f89d60238ad7387432d34167717 into a4854849cd634dd2b34f5b268d21cf0a(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:20,657 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:20,657 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., storeName=a5436f89d60238ad7387432d34167717/C, priority=11, startTime=1732137320531; duration=0sec 2024-11-20T21:15:20,657 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:20,657 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:C 2024-11-20T21:15:20,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T21:15:20,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on a5436f89d60238ad7387432d34167717 2024-11-20T21:15:20,711 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a5436f89d60238ad7387432d34167717 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T21:15:20,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=A 2024-11-20T21:15:20,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:20,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=B 2024-11-20T21:15:20,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:20,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=C 2024-11-20T21:15:20,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:20,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/14a323b27fe644ff967fbdb8595a88aa is 50, key is test_row_0/A:col10/1732137320390/Put/seqid=0 2024-11-20T21:15:20,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742198_1374 (size=14741) 2024-11-20T21:15:20,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/14a323b27fe644ff967fbdb8595a88aa 2024-11-20T21:15:20,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/8b2a4a2790be46af9ef4fa9d7dde3148 is 50, key is test_row_0/B:col10/1732137320390/Put/seqid=0 2024-11-20T21:15:20,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T21:15:20,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:20,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:20,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137380747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:20,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:20,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:20,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:20,756 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:20,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:20,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:20,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:20,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137380749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:20,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137380748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:20,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:20,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:20,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137380750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:20,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:20,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137380752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:20,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,759 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:20,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,759 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:20,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742199_1375 (size=12301) 2024-11-20T21:15:20,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:20,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137380867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:20,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:20,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:20,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137380868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:20,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:20,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137380868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:20,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:20,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:20,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137380868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:20,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:20,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:20,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137380869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:20,873 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:20,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T21:15:20,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:20,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:20,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:20,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:20,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:20,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:20,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T21:15:21,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T21:15:21,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:21,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:21,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:21,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:21,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:21,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:21,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:21,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137381077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:21,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:21,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:21,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137381077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:21,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:21,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:21,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137381078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:21,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:21,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137381078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:21,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:21,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137381078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:21,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:21,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/8b2a4a2790be46af9ef4fa9d7dde3148 2024-11-20T21:15:21,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/a93640f08324472e90ea2d04052502e6 is 50, key is test_row_0/C:col10/1732137320390/Put/seqid=0 2024-11-20T21:15:21,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T21:15:21,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:21,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:21,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:21,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:21,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:21,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:21,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T21:15:21,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742200_1376 (size=12301) 2024-11-20T21:15:21,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T21:15:21,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:21,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:21,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:21,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:21,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:21,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:21,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:21,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60014 deadline: 1732137381386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:21,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:21,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60070 deadline: 1732137381387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:21,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:21,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:21,388 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:21,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:21,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:21,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:21,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60062 deadline: 1732137381387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:21,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:21,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:21,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:21,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60056 deadline: 1732137381387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:21,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:21,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:21,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:21,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:21,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:21,391 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:21,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:60072 deadline: 1732137381387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:21,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:21,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:21,391 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:21,509 INFO [Thread-1216 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1216. 2024-11-20T21:15:21,509 DEBUG [Thread-1216 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:15:21,509 DEBUG [Thread-1216 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:21,509 DEBUG [Thread-1216 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:21,509 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:15:21,512 INFO [Thread-1218 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1218. 2024-11-20T21:15:21,512 DEBUG [Thread-1218 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:15:21,512 DEBUG [Thread-1218 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:21,513 DEBUG [Thread-1218 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:21,513 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:15:21,514 INFO [Thread-1224 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1224. 2024-11-20T21:15:21,514 DEBUG [Thread-1224 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:15:21,514 DEBUG [Thread-1224 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:21,514 DEBUG [Thread-1224 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:21,514 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:15:21,515 INFO [Thread-1222 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1222. 2024-11-20T21:15:21,515 DEBUG [Thread-1222 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:15:21,515 DEBUG [Thread-1222 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:21,515 DEBUG [Thread-1222 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:21,515 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:15:21,517 INFO [Thread-1220 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1220. 2024-11-20T21:15:21,517 DEBUG [Thread-1220 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:15:21,517 DEBUG [Thread-1220 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:21,517 DEBUG [Thread-1220 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:21,517 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:15:21,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T21:15:21,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:21,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:21,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:21,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:21,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:21,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:21,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/a93640f08324472e90ea2d04052502e6 2024-11-20T21:15:21,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/14a323b27fe644ff967fbdb8595a88aa as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/14a323b27fe644ff967fbdb8595a88aa 2024-11-20T21:15:21,638 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/14a323b27fe644ff967fbdb8595a88aa, entries=200, sequenceid=517, filesize=14.4 K 2024-11-20T21:15:21,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/8b2a4a2790be46af9ef4fa9d7dde3148 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/8b2a4a2790be46af9ef4fa9d7dde3148 2024-11-20T21:15:21,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/8b2a4a2790be46af9ef4fa9d7dde3148, entries=150, sequenceid=517, filesize=12.0 K 2024-11-20T21:15:21,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/a93640f08324472e90ea2d04052502e6 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/a93640f08324472e90ea2d04052502e6 2024-11-20T21:15:21,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/a93640f08324472e90ea2d04052502e6, entries=150, sequenceid=517, filesize=12.0 K 2024-11-20T21:15:21,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for a5436f89d60238ad7387432d34167717 in 934ms, sequenceid=517, compaction requested=true 2024-11-20T21:15:21,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:21,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:15:21,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:21,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:15:21,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:21,645 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:21,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a5436f89d60238ad7387432d34167717:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:15:21,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:21,645 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:21,646 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40535 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:21,646 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:21,646 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): a5436f89d60238ad7387432d34167717/A is initiating minor compaction (all files) 2024-11-20T21:15:21,646 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): a5436f89d60238ad7387432d34167717/B is initiating minor compaction (all files) 2024-11-20T21:15:21,646 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5436f89d60238ad7387432d34167717/A in TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:21,646 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a5436f89d60238ad7387432d34167717/B in TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:21,646 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/3792b5a0aebe443c84cffc0ac4588e44, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/f4da54c913114f6f9e8960e72ac8ffd1, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/14a323b27fe644ff967fbdb8595a88aa] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp, totalSize=39.6 K 2024-11-20T21:15:21,646 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/9fa4727c2e174ef88a5f1994be54ffcd, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/3085138e0aa743f689069d684d854ed5, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/8b2a4a2790be46af9ef4fa9d7dde3148] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp, totalSize=37.2 K 2024-11-20T21:15:21,646 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3792b5a0aebe443c84cffc0ac4588e44, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1732137319135 2024-11-20T21:15:21,646 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fa4727c2e174ef88a5f1994be54ffcd, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=476, earliestPutTs=1732137319135 2024-11-20T21:15:21,647 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4da54c913114f6f9e8960e72ac8ffd1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1732137319189 2024-11-20T21:15:21,647 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 3085138e0aa743f689069d684d854ed5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1732137319189 2024-11-20T21:15:21,647 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14a323b27fe644ff967fbdb8595a88aa, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=517, earliestPutTs=1732137320386 2024-11-20T21:15:21,647 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b2a4a2790be46af9ef4fa9d7dde3148, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=517, earliestPutTs=1732137320386 2024-11-20T21:15:21,653 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5436f89d60238ad7387432d34167717#B#compaction#324 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:21,653 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/8689378cabc9406b817b8ed211f77269 is 50, key is test_row_0/B:col10/1732137320390/Put/seqid=0 2024-11-20T21:15:21,654 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a5436f89d60238ad7387432d34167717#A#compaction#325 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:21,654 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/f1646e9a74ee4a259efc97b3bfbadd02 is 50, key is test_row_0/A:col10/1732137320390/Put/seqid=0 2024-11-20T21:15:21,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742201_1377 (size=13595) 2024-11-20T21:15:21,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742202_1378 (size=13595) 2024-11-20T21:15:21,663 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/8689378cabc9406b817b8ed211f77269 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/8689378cabc9406b817b8ed211f77269 2024-11-20T21:15:21,665 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/f1646e9a74ee4a259efc97b3bfbadd02 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/f1646e9a74ee4a259efc97b3bfbadd02 2024-11-20T21:15:21,668 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5436f89d60238ad7387432d34167717/B of a5436f89d60238ad7387432d34167717 into 8689378cabc9406b817b8ed211f77269(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:21,668 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:21,668 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., storeName=a5436f89d60238ad7387432d34167717/B, priority=13, startTime=1732137321645; duration=0sec 2024-11-20T21:15:21,668 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:21,668 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:B 2024-11-20T21:15:21,668 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-20T21:15:21,669 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-20T21:15:21,669 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a5436f89d60238ad7387432d34167717/A of a5436f89d60238ad7387432d34167717 into f1646e9a74ee4a259efc97b3bfbadd02(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:21,669 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-20T21:15:21,669 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:21,669 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., storeName=a5436f89d60238ad7387432d34167717/A, priority=13, startTime=1732137321645; duration=0sec 2024-11-20T21:15:21,669 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. because compaction request was cancelled 2024-11-20T21:15:21,669 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:C 2024-11-20T21:15:21,669 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:21,669 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a5436f89d60238ad7387432d34167717:A 2024-11-20T21:15:21,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-20T21:15:21,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:21,680 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing a5436f89d60238ad7387432d34167717 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T21:15:21,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=A 2024-11-20T21:15:21,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:21,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=B 2024-11-20T21:15:21,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:21,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=C 2024-11-20T21:15:21,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:21,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/4a39d7765b8148a8b65ec8776dff5c37 is 50, key is test_row_0/A:col10/1732137320744/Put/seqid=0 2024-11-20T21:15:21,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742203_1379 (size=12301) 2024-11-20T21:15:21,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T21:15:21,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on a5436f89d60238ad7387432d34167717 2024-11-20T21:15:21,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. as already flushing 2024-11-20T21:15:21,898 INFO [Thread-1209 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1209. 2024-11-20T21:15:21,898 INFO [Thread-1205 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1205. 2024-11-20T21:15:21,898 INFO [Thread-1211 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1211. 2024-11-20T21:15:21,898 DEBUG [Thread-1209 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:15:21,898 DEBUG [Thread-1205 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:15:21,898 DEBUG [Thread-1211 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:15:21,898 DEBUG [Thread-1205 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:21,898 DEBUG [Thread-1209 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:21,899 DEBUG [Thread-1211 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:21,899 DEBUG [Thread-1205 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:21,899 DEBUG [Thread-1209 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:21,899 DEBUG [Thread-1211 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:21,899 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:15:21,899 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:15:21,899 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:15:21,900 INFO [Thread-1207 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1207. 2024-11-20T21:15:21,900 DEBUG [Thread-1207 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:15:21,900 DEBUG [Thread-1207 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:21,900 DEBUG [Thread-1207 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:21,900 INFO [Thread-1213 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1213. 2024-11-20T21:15:21,900 DEBUG [Thread-1213 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:15:21,900 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:15:21,900 DEBUG [Thread-1213 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:21,900 DEBUG [Thread-1213 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:21,900 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:15:22,093 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/4a39d7765b8148a8b65ec8776dff5c37 2024-11-20T21:15:22,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/d8a4cf3dfa4c49bc9a8b8798aa74a6ac is 50, key is test_row_0/B:col10/1732137320744/Put/seqid=0 2024-11-20T21:15:22,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742204_1380 (size=12301) 2024-11-20T21:15:22,506 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/d8a4cf3dfa4c49bc9a8b8798aa74a6ac 2024-11-20T21:15:22,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/158f19033ee44698bcd3d1657f40a1f2 is 50, key is test_row_0/C:col10/1732137320744/Put/seqid=0 2024-11-20T21:15:22,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742205_1381 (size=12301) 2024-11-20T21:15:22,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T21:15:22,922 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=539 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/158f19033ee44698bcd3d1657f40a1f2 2024-11-20T21:15:22,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/4a39d7765b8148a8b65ec8776dff5c37 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/4a39d7765b8148a8b65ec8776dff5c37 2024-11-20T21:15:22,930 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/4a39d7765b8148a8b65ec8776dff5c37, entries=150, sequenceid=539, filesize=12.0 K 2024-11-20T21:15:22,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/d8a4cf3dfa4c49bc9a8b8798aa74a6ac as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/d8a4cf3dfa4c49bc9a8b8798aa74a6ac 2024-11-20T21:15:22,934 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/d8a4cf3dfa4c49bc9a8b8798aa74a6ac, entries=150, sequenceid=539, filesize=12.0 K 2024-11-20T21:15:22,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/158f19033ee44698bcd3d1657f40a1f2 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/158f19033ee44698bcd3d1657f40a1f2 2024-11-20T21:15:22,937 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/158f19033ee44698bcd3d1657f40a1f2, entries=150, sequenceid=539, filesize=12.0 K 2024-11-20T21:15:22,938 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=33.54 KB/34350 for a5436f89d60238ad7387432d34167717 in 1258ms, sequenceid=539, compaction requested=true 2024-11-20T21:15:22,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for a5436f89d60238ad7387432d34167717: 2024-11-20T21:15:22,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:22,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-20T21:15:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-20T21:15:22,941 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-20T21:15:22,941 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3380 sec 2024-11-20T21:15:22,942 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.3430 sec 2024-11-20T21:15:24,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-20T21:15:24,778 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:15:24,778 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T21:15:24,778 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-11-20T21:15:24,778 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-20T21:15:24,778 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-20T21:15:24,778 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-20T21:15:24,778 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 84 2024-11-20T21:15:24,778 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T21:15:24,778 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1942 2024-11-20T21:15:24,778 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1933 2024-11-20T21:15:24,778 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1928 2024-11-20T21:15:24,778 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1941 2024-11-20T21:15:24,778 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1936 2024-11-20T21:15:24,778 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T21:15:24,778 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T21:15:24,778 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T21:15:24,778 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:24,779 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:24,779 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:15:24,780 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-20T21:15:24,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T21:15:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T21:15:24,783 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732137324783"}]},"ts":"1732137324783"} 2024-11-20T21:15:24,785 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T21:15:24,785 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T21:15:24,786 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T21:15:24,788 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5436f89d60238ad7387432d34167717, UNASSIGN}] 2024-11-20T21:15:24,789 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5436f89d60238ad7387432d34167717, UNASSIGN 2024-11-20T21:15:24,790 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=a5436f89d60238ad7387432d34167717, regionState=CLOSING, regionLocation=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:24,791 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=79, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5436f89d60238ad7387432d34167717, UNASSIGN because future has completed 2024-11-20T21:15:24,791 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T21:15:24,791 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536}] 2024-11-20T21:15:24,815 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T21:15:24,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T21:15:24,945 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(122): Close a5436f89d60238ad7387432d34167717 2024-11-20T21:15:24,945 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T21:15:24,945 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1722): Closing a5436f89d60238ad7387432d34167717, disabling compactions & flushes 2024-11-20T21:15:24,945 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:24,945 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:24,945 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. after waiting 0 ms 2024-11-20T21:15:24,945 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:24,945 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(2902): Flushing a5436f89d60238ad7387432d34167717 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-20T21:15:24,952 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=A 2024-11-20T21:15:24,952 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:24,952 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=B 2024-11-20T21:15:24,952 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:24,953 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK a5436f89d60238ad7387432d34167717, store=C 2024-11-20T21:15:24,953 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:24,958 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/b7084acc33be44e38e8c4d5073b3d383 is 50, key is test_row_0/A:col10/1732137321897/Put/seqid=0 2024-11-20T21:15:24,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742206_1382 (size=12301) 2024-11-20T21:15:24,971 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/b7084acc33be44e38e8c4d5073b3d383 2024-11-20T21:15:24,986 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/3cb1b72903e949bea406c534572449dc is 50, key is test_row_0/B:col10/1732137321897/Put/seqid=0 2024-11-20T21:15:24,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742207_1383 (size=12301) 2024-11-20T21:15:25,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T21:15:25,395 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/3cb1b72903e949bea406c534572449dc 2024-11-20T21:15:25,408 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/27b05e061a6e4768b0e01e366a3c04a5 is 50, key is test_row_0/C:col10/1732137321897/Put/seqid=0 2024-11-20T21:15:25,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T21:15:25,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742208_1384 (size=12301) 2024-11-20T21:15:25,831 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=547 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/27b05e061a6e4768b0e01e366a3c04a5 2024-11-20T21:15:25,836 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/A/b7084acc33be44e38e8c4d5073b3d383 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/b7084acc33be44e38e8c4d5073b3d383 2024-11-20T21:15:25,840 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/b7084acc33be44e38e8c4d5073b3d383, entries=150, sequenceid=547, filesize=12.0 K 2024-11-20T21:15:25,841 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/B/3cb1b72903e949bea406c534572449dc as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/3cb1b72903e949bea406c534572449dc 2024-11-20T21:15:25,844 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/3cb1b72903e949bea406c534572449dc, entries=150, sequenceid=547, filesize=12.0 K 2024-11-20T21:15:25,845 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/.tmp/C/27b05e061a6e4768b0e01e366a3c04a5 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/27b05e061a6e4768b0e01e366a3c04a5 2024-11-20T21:15:25,849 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/27b05e061a6e4768b0e01e366a3c04a5, entries=150, sequenceid=547, filesize=12.0 K 2024-11-20T21:15:25,850 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for a5436f89d60238ad7387432d34167717 in 905ms, sequenceid=547, compaction requested=true 2024-11-20T21:15:25,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/22eb65125c4848ca896e55d3dddf4a25, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/971ad6c2479040448e811dad71a40756, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/0bfb5fb575574f229c3550f3640afb7d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/f500248ec1d341808dc02849068d0d55, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/bd8ebb8d74e84a35bbbfb3a4222a2640, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/483102a936394eaba2f0861c17e607fa, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/02fa269e7dbf420aa988c4e40f1cad7c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/dc960f2c82884e0585408f3bee6d0b36, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/3cdc7dd469d6495c903c833605d918a5, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/4e72287192744c80bf9436324a58bc0a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/1bc13260cc2f4bf285e65173ad1b8eaf, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/7cc6d337dcb74544bffe0e935ef31170, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/29d3d1ac07f04dc0825ee37549dcd2a8, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/57f027aea9e24e19805fc688033c891a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/5bf4f451f2eb4a308f5321e265f6578d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/8fa5706069e3468a97e144e30f37cb31, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/e0675079aade4e0e950459fe05aeef10, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/c6ebeb145a194c00b379c2d0f4f9a18d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/3899816d1ef246288a2896b9be789274, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/dadb85f6d6994133899ef6295aa17f82, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/744d6a6172814b42b4d91d3588dd2b3d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/b0afda355419416d8d1841a3b19a9b1f, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/96d9090a665e47cf95fdc40ab40e1704, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/f0b508fbcf5e4760b2afdf8560127254, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/a3ee8916fc23480cad19fc55cc9c1b78, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/3ba9f92da1f84003b03e105609930d33, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/f45acd17fe054fe18b1bad6fa7247075, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/9ea6561546de44aab4eb36daab164d82, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/859d6a667c564ceebb58aa0012e32be3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/b07f0c62914742f791cb7fe2c3a92eb2, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/107e7d97a45f4215bdba5cfbd7c08c01, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/aad7e7ace5414dc98e73389652bf76d2, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/2ddd671d940648559c561f5fae1b8896, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/e77dc428c4024908be1835de124fea7d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/3792b5a0aebe443c84cffc0ac4588e44, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/aad404b478804682a53c74a9503a5c64, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/f4da54c913114f6f9e8960e72ac8ffd1, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/14a323b27fe644ff967fbdb8595a88aa] to archive 2024-11-20T21:15:25,856 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T21:15:25,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/22eb65125c4848ca896e55d3dddf4a25 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/22eb65125c4848ca896e55d3dddf4a25 2024-11-20T21:15:25,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/971ad6c2479040448e811dad71a40756 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/971ad6c2479040448e811dad71a40756 2024-11-20T21:15:25,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/0bfb5fb575574f229c3550f3640afb7d to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/0bfb5fb575574f229c3550f3640afb7d 2024-11-20T21:15:25,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/f500248ec1d341808dc02849068d0d55 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/f500248ec1d341808dc02849068d0d55 2024-11-20T21:15:25,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/bd8ebb8d74e84a35bbbfb3a4222a2640 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/bd8ebb8d74e84a35bbbfb3a4222a2640 2024-11-20T21:15:25,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/483102a936394eaba2f0861c17e607fa to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/483102a936394eaba2f0861c17e607fa 2024-11-20T21:15:25,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/02fa269e7dbf420aa988c4e40f1cad7c to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/02fa269e7dbf420aa988c4e40f1cad7c 2024-11-20T21:15:25,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/dc960f2c82884e0585408f3bee6d0b36 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/dc960f2c82884e0585408f3bee6d0b36 2024-11-20T21:15:25,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/3cdc7dd469d6495c903c833605d918a5 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/3cdc7dd469d6495c903c833605d918a5 2024-11-20T21:15:25,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/4e72287192744c80bf9436324a58bc0a to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/4e72287192744c80bf9436324a58bc0a 2024-11-20T21:15:25,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/1bc13260cc2f4bf285e65173ad1b8eaf to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/1bc13260cc2f4bf285e65173ad1b8eaf 2024-11-20T21:15:25,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/7cc6d337dcb74544bffe0e935ef31170 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/7cc6d337dcb74544bffe0e935ef31170 2024-11-20T21:15:25,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/29d3d1ac07f04dc0825ee37549dcd2a8 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/29d3d1ac07f04dc0825ee37549dcd2a8 2024-11-20T21:15:25,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/57f027aea9e24e19805fc688033c891a to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/57f027aea9e24e19805fc688033c891a 2024-11-20T21:15:25,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/5bf4f451f2eb4a308f5321e265f6578d to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/5bf4f451f2eb4a308f5321e265f6578d 2024-11-20T21:15:25,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/8fa5706069e3468a97e144e30f37cb31 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/8fa5706069e3468a97e144e30f37cb31 2024-11-20T21:15:25,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/e0675079aade4e0e950459fe05aeef10 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/e0675079aade4e0e950459fe05aeef10 2024-11-20T21:15:25,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/c6ebeb145a194c00b379c2d0f4f9a18d to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/c6ebeb145a194c00b379c2d0f4f9a18d 2024-11-20T21:15:25,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/3899816d1ef246288a2896b9be789274 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/3899816d1ef246288a2896b9be789274 2024-11-20T21:15:25,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/dadb85f6d6994133899ef6295aa17f82 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/dadb85f6d6994133899ef6295aa17f82 2024-11-20T21:15:25,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/744d6a6172814b42b4d91d3588dd2b3d to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/744d6a6172814b42b4d91d3588dd2b3d 2024-11-20T21:15:25,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/b0afda355419416d8d1841a3b19a9b1f to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/b0afda355419416d8d1841a3b19a9b1f 2024-11-20T21:15:25,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/96d9090a665e47cf95fdc40ab40e1704 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/96d9090a665e47cf95fdc40ab40e1704 2024-11-20T21:15:25,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/f0b508fbcf5e4760b2afdf8560127254 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/f0b508fbcf5e4760b2afdf8560127254 2024-11-20T21:15:25,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/a3ee8916fc23480cad19fc55cc9c1b78 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/a3ee8916fc23480cad19fc55cc9c1b78 2024-11-20T21:15:25,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/3ba9f92da1f84003b03e105609930d33 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/3ba9f92da1f84003b03e105609930d33 2024-11-20T21:15:25,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/f45acd17fe054fe18b1bad6fa7247075 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/f45acd17fe054fe18b1bad6fa7247075 2024-11-20T21:15:25,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/9ea6561546de44aab4eb36daab164d82 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/9ea6561546de44aab4eb36daab164d82 2024-11-20T21:15:25,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/859d6a667c564ceebb58aa0012e32be3 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/859d6a667c564ceebb58aa0012e32be3 2024-11-20T21:15:25,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/b07f0c62914742f791cb7fe2c3a92eb2 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/b07f0c62914742f791cb7fe2c3a92eb2 2024-11-20T21:15:25,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/107e7d97a45f4215bdba5cfbd7c08c01 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/107e7d97a45f4215bdba5cfbd7c08c01 2024-11-20T21:15:25,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/aad7e7ace5414dc98e73389652bf76d2 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/aad7e7ace5414dc98e73389652bf76d2 2024-11-20T21:15:25,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/2ddd671d940648559c561f5fae1b8896 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/2ddd671d940648559c561f5fae1b8896 2024-11-20T21:15:25,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/e77dc428c4024908be1835de124fea7d to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/e77dc428c4024908be1835de124fea7d 2024-11-20T21:15:25,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/3792b5a0aebe443c84cffc0ac4588e44 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/3792b5a0aebe443c84cffc0ac4588e44 2024-11-20T21:15:25,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/aad404b478804682a53c74a9503a5c64 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/aad404b478804682a53c74a9503a5c64 2024-11-20T21:15:25,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/f4da54c913114f6f9e8960e72ac8ffd1 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/f4da54c913114f6f9e8960e72ac8ffd1 2024-11-20T21:15:25,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/14a323b27fe644ff967fbdb8595a88aa to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/14a323b27fe644ff967fbdb8595a88aa 2024-11-20T21:15:25,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/becc28bd1a9d4842a618526d7ccc5437, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/9083ba1e3518460c9603816495734b47, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/7181a63626d44e91979a2be28ae61a11, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/42a89dd70fae426d8b33321b321b9bcd, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/72e6816f57744239819ab17ec9e0719d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/a39b9beb2e5b44d8bab4706e9a3aff43, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/31be0d6b31a84d99bccbcb80c8cd17f2, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/71af558105894c1db085ae4ddb905cbe, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/710945fc237b42e3b066ee05589b1a16, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/fb3a9e62c05b47998aeef67ece01394b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/8dedd7ee450c404a8677b8a99412dc65, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/2e5c49f4c88240818f19110d986e5c51, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/68c9f331e2364ad89de8e68451ab0494, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/044f8fb2ac334612b3e1aa4404fa42f3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/4971a6f9423c4ef6bfe9178b4f54c8b8, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/5a694f8130f74b15a62a55c720a43a10, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/5479bf73d5344b67ba50f205b56f0d81, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/0fe2d34cabc0420b8d05b344b6f6e2c5, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/f34b88505c30493185e32880d6d72792, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/b27391d81df84f1a808bacf0490dfeae, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/089221c56d954ca4a4e7602ca7920774, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/0c6c7a3d7c404d629a7faf41cd177f3b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/0e4300d792824da6acfb9b4480508905, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/64a8353ff03a4a55a52ee390aca73aff, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/3f1b017b7ff14015aebb113469e8a549, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/7a27248eb7db4862a86aaf3ed54e70bb, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/08f65886949b467c840971e94798d35e, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/904c6362a9fb4123a3de2213ea16d06d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/b57633b3e5a24149a4bd4f7e049a4773, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/45d6991ba512433baff63ce806dcaf89, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/6cd3e609eeab49d69d691b4dcfa88083, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/77509657f9d146978ea66dc34cde3c03, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/f94bc79718da4237a82f5b03f0d15f19, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/2974c913612d4fdbb67178cb39645993, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/9fa4727c2e174ef88a5f1994be54ffcd, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/e42c90442f5c47168f9df624152c0f34, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/3085138e0aa743f689069d684d854ed5, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/8b2a4a2790be46af9ef4fa9d7dde3148] to archive 2024-11-20T21:15:25,898 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T21:15:25,899 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/becc28bd1a9d4842a618526d7ccc5437 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/becc28bd1a9d4842a618526d7ccc5437 2024-11-20T21:15:25,900 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/9083ba1e3518460c9603816495734b47 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/9083ba1e3518460c9603816495734b47 2024-11-20T21:15:25,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/7181a63626d44e91979a2be28ae61a11 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/7181a63626d44e91979a2be28ae61a11 2024-11-20T21:15:25,902 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/42a89dd70fae426d8b33321b321b9bcd to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/42a89dd70fae426d8b33321b321b9bcd 2024-11-20T21:15:25,903 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/72e6816f57744239819ab17ec9e0719d to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/72e6816f57744239819ab17ec9e0719d 2024-11-20T21:15:25,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/a39b9beb2e5b44d8bab4706e9a3aff43 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/a39b9beb2e5b44d8bab4706e9a3aff43 2024-11-20T21:15:25,906 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/31be0d6b31a84d99bccbcb80c8cd17f2 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/31be0d6b31a84d99bccbcb80c8cd17f2 2024-11-20T21:15:25,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/71af558105894c1db085ae4ddb905cbe to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/71af558105894c1db085ae4ddb905cbe 2024-11-20T21:15:25,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/710945fc237b42e3b066ee05589b1a16 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/710945fc237b42e3b066ee05589b1a16 2024-11-20T21:15:25,910 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/fb3a9e62c05b47998aeef67ece01394b to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/fb3a9e62c05b47998aeef67ece01394b 2024-11-20T21:15:25,911 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/8dedd7ee450c404a8677b8a99412dc65 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/8dedd7ee450c404a8677b8a99412dc65 2024-11-20T21:15:25,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/2e5c49f4c88240818f19110d986e5c51 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/2e5c49f4c88240818f19110d986e5c51 2024-11-20T21:15:25,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/68c9f331e2364ad89de8e68451ab0494 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/68c9f331e2364ad89de8e68451ab0494 2024-11-20T21:15:25,913 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/044f8fb2ac334612b3e1aa4404fa42f3 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/044f8fb2ac334612b3e1aa4404fa42f3 2024-11-20T21:15:25,915 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/4971a6f9423c4ef6bfe9178b4f54c8b8 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/4971a6f9423c4ef6bfe9178b4f54c8b8 2024-11-20T21:15:25,916 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/5a694f8130f74b15a62a55c720a43a10 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/5a694f8130f74b15a62a55c720a43a10 2024-11-20T21:15:25,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T21:15:25,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/5479bf73d5344b67ba50f205b56f0d81 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/5479bf73d5344b67ba50f205b56f0d81 2024-11-20T21:15:25,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/0fe2d34cabc0420b8d05b344b6f6e2c5 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/0fe2d34cabc0420b8d05b344b6f6e2c5 2024-11-20T21:15:25,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/f34b88505c30493185e32880d6d72792 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/f34b88505c30493185e32880d6d72792 2024-11-20T21:15:25,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/b27391d81df84f1a808bacf0490dfeae to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/b27391d81df84f1a808bacf0490dfeae 2024-11-20T21:15:25,921 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/089221c56d954ca4a4e7602ca7920774 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/089221c56d954ca4a4e7602ca7920774 2024-11-20T21:15:25,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/0c6c7a3d7c404d629a7faf41cd177f3b to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/0c6c7a3d7c404d629a7faf41cd177f3b 2024-11-20T21:15:25,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/0e4300d792824da6acfb9b4480508905 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/0e4300d792824da6acfb9b4480508905 2024-11-20T21:15:25,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/64a8353ff03a4a55a52ee390aca73aff to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/64a8353ff03a4a55a52ee390aca73aff 2024-11-20T21:15:25,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/3f1b017b7ff14015aebb113469e8a549 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/3f1b017b7ff14015aebb113469e8a549 2024-11-20T21:15:25,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/7a27248eb7db4862a86aaf3ed54e70bb to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/7a27248eb7db4862a86aaf3ed54e70bb 2024-11-20T21:15:25,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/08f65886949b467c840971e94798d35e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/08f65886949b467c840971e94798d35e 2024-11-20T21:15:25,928 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/904c6362a9fb4123a3de2213ea16d06d to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/904c6362a9fb4123a3de2213ea16d06d 2024-11-20T21:15:25,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/b57633b3e5a24149a4bd4f7e049a4773 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/b57633b3e5a24149a4bd4f7e049a4773 2024-11-20T21:15:25,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/45d6991ba512433baff63ce806dcaf89 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/45d6991ba512433baff63ce806dcaf89 2024-11-20T21:15:25,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/6cd3e609eeab49d69d691b4dcfa88083 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/6cd3e609eeab49d69d691b4dcfa88083 2024-11-20T21:15:25,931 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/77509657f9d146978ea66dc34cde3c03 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/77509657f9d146978ea66dc34cde3c03 2024-11-20T21:15:25,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/f94bc79718da4237a82f5b03f0d15f19 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/f94bc79718da4237a82f5b03f0d15f19 2024-11-20T21:15:25,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/2974c913612d4fdbb67178cb39645993 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/2974c913612d4fdbb67178cb39645993 2024-11-20T21:15:25,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/9fa4727c2e174ef88a5f1994be54ffcd to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/9fa4727c2e174ef88a5f1994be54ffcd 2024-11-20T21:15:25,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/e42c90442f5c47168f9df624152c0f34 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/e42c90442f5c47168f9df624152c0f34 2024-11-20T21:15:25,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/3085138e0aa743f689069d684d854ed5 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/3085138e0aa743f689069d684d854ed5 2024-11-20T21:15:25,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/8b2a4a2790be46af9ef4fa9d7dde3148 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/8b2a4a2790be46af9ef4fa9d7dde3148 2024-11-20T21:15:25,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/c6f09b7ca26542ea8f7051457fe10833, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/4d51eec789444326874535f0b4b91cae, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/63ec8b5a3dda4839ba11ef05e695f09e, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/f38d2ada4ad44ec98729fe6a5c6db2eb, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/6abe23639d1e4475894b5e928984b345, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/9fe9cd08e1cc498b9ee65ea97fd83315, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/12961304a58e4f5bb6e55eb1802a7f9e, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/45cd2ddc31e547539d6c2f2e16d848d3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/3bec09d5b4374e9dbd6747ebd7d32d7b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/c1ec87f4ae604f74bcfe236a20d25b9f, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/c7ba9958767f46859bb8581a9487a238, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/2bbc915e19f94f00aa152f5bc9c1e79b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/1cd33031772444c68e6579d7b356ec12, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/17c6eb96724449baa7a6f75be2401ba9, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/db9909c20c4947af8fd9873dd8d53657, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/ab62023f38284f65a9ba65b306c0dc71, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/dd26f54c1f3b470fb13c7aed72066b56, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/a78bb1acdba04ca3ad54f0202ee124df, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/590d5a27f472465fab219b854ca82256, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/b2c1660c349e4e0aaec83995d0afb421, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/93134d06f1c144ad9dd978840ab74677, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/8d5a959491ea4b7d96086d01bad28945, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/22feb88e3a4a49c5828281bca3aad9d3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/965f0f041a304aadaf64caf287452253, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/be29ccbc017d47cfb9482480e49bc778, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/c26699a95ac7473d9c6b98bcb8fc0048, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/d5a926179c3045b2bddfae6f3cbfeb10, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/77abb8997ce94ce1aff5243d6f742d97, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/2efc2c1f3e144b9ab88e715f217d1545, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/58e6c37802074388850e8a82ae5a9578, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/adc246b373c940ec88e376ab4997bbf0, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/0a22a2d6f44349c489350f9e5219c008, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/b1aaab2924494e52a30298b1dbb42a32, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/a49a9b19cbf848ee802090f6fa025e73, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/61642adcb0e94096ba9056110f597c84, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/f9fd07b8dfed4ac7a368f0ca9a24a075] to archive 2024-11-20T21:15:25,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T21:15:25,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/c6f09b7ca26542ea8f7051457fe10833 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/c6f09b7ca26542ea8f7051457fe10833 2024-11-20T21:15:25,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/4d51eec789444326874535f0b4b91cae to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/4d51eec789444326874535f0b4b91cae 2024-11-20T21:15:25,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/63ec8b5a3dda4839ba11ef05e695f09e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/63ec8b5a3dda4839ba11ef05e695f09e 2024-11-20T21:15:25,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/f38d2ada4ad44ec98729fe6a5c6db2eb to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/f38d2ada4ad44ec98729fe6a5c6db2eb 2024-11-20T21:15:25,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/6abe23639d1e4475894b5e928984b345 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/6abe23639d1e4475894b5e928984b345 2024-11-20T21:15:25,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/9fe9cd08e1cc498b9ee65ea97fd83315 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/9fe9cd08e1cc498b9ee65ea97fd83315 2024-11-20T21:15:25,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/12961304a58e4f5bb6e55eb1802a7f9e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/12961304a58e4f5bb6e55eb1802a7f9e 2024-11-20T21:15:25,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/45cd2ddc31e547539d6c2f2e16d848d3 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/45cd2ddc31e547539d6c2f2e16d848d3 2024-11-20T21:15:25,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/3bec09d5b4374e9dbd6747ebd7d32d7b to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/3bec09d5b4374e9dbd6747ebd7d32d7b 2024-11-20T21:15:25,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/c1ec87f4ae604f74bcfe236a20d25b9f to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/c1ec87f4ae604f74bcfe236a20d25b9f 2024-11-20T21:15:25,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/c7ba9958767f46859bb8581a9487a238 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/c7ba9958767f46859bb8581a9487a238 2024-11-20T21:15:25,950 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/2bbc915e19f94f00aa152f5bc9c1e79b to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/2bbc915e19f94f00aa152f5bc9c1e79b 2024-11-20T21:15:25,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/1cd33031772444c68e6579d7b356ec12 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/1cd33031772444c68e6579d7b356ec12 2024-11-20T21:15:25,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/17c6eb96724449baa7a6f75be2401ba9 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/17c6eb96724449baa7a6f75be2401ba9 2024-11-20T21:15:25,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/db9909c20c4947af8fd9873dd8d53657 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/db9909c20c4947af8fd9873dd8d53657 2024-11-20T21:15:25,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/ab62023f38284f65a9ba65b306c0dc71 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/ab62023f38284f65a9ba65b306c0dc71 2024-11-20T21:15:25,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/dd26f54c1f3b470fb13c7aed72066b56 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/dd26f54c1f3b470fb13c7aed72066b56 2024-11-20T21:15:25,955 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/a78bb1acdba04ca3ad54f0202ee124df to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/a78bb1acdba04ca3ad54f0202ee124df 2024-11-20T21:15:25,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/590d5a27f472465fab219b854ca82256 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/590d5a27f472465fab219b854ca82256 2024-11-20T21:15:25,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/b2c1660c349e4e0aaec83995d0afb421 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/b2c1660c349e4e0aaec83995d0afb421 2024-11-20T21:15:25,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/93134d06f1c144ad9dd978840ab74677 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/93134d06f1c144ad9dd978840ab74677 2024-11-20T21:15:25,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/8d5a959491ea4b7d96086d01bad28945 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/8d5a959491ea4b7d96086d01bad28945 2024-11-20T21:15:25,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/22feb88e3a4a49c5828281bca3aad9d3 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/22feb88e3a4a49c5828281bca3aad9d3 2024-11-20T21:15:25,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/965f0f041a304aadaf64caf287452253 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/965f0f041a304aadaf64caf287452253 2024-11-20T21:15:25,961 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/be29ccbc017d47cfb9482480e49bc778 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/be29ccbc017d47cfb9482480e49bc778 2024-11-20T21:15:25,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/c26699a95ac7473d9c6b98bcb8fc0048 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/c26699a95ac7473d9c6b98bcb8fc0048 2024-11-20T21:15:25,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/d5a926179c3045b2bddfae6f3cbfeb10 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/d5a926179c3045b2bddfae6f3cbfeb10 2024-11-20T21:15:25,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/77abb8997ce94ce1aff5243d6f742d97 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/77abb8997ce94ce1aff5243d6f742d97 2024-11-20T21:15:25,964 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/2efc2c1f3e144b9ab88e715f217d1545 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/2efc2c1f3e144b9ab88e715f217d1545 2024-11-20T21:15:25,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/58e6c37802074388850e8a82ae5a9578 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/58e6c37802074388850e8a82ae5a9578 2024-11-20T21:15:25,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/adc246b373c940ec88e376ab4997bbf0 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/adc246b373c940ec88e376ab4997bbf0 2024-11-20T21:15:25,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/0a22a2d6f44349c489350f9e5219c008 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/0a22a2d6f44349c489350f9e5219c008 2024-11-20T21:15:25,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/b1aaab2924494e52a30298b1dbb42a32 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/b1aaab2924494e52a30298b1dbb42a32 2024-11-20T21:15:25,968 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/a49a9b19cbf848ee802090f6fa025e73 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/a49a9b19cbf848ee802090f6fa025e73 2024-11-20T21:15:25,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/61642adcb0e94096ba9056110f597c84 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/61642adcb0e94096ba9056110f597c84 2024-11-20T21:15:25,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/f9fd07b8dfed4ac7a368f0ca9a24a075 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/f9fd07b8dfed4ac7a368f0ca9a24a075 2024-11-20T21:15:25,973 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/recovered.edits/550.seqid, newMaxSeqId=550, maxSeqId=1 2024-11-20T21:15:25,974 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. 2024-11-20T21:15:25,974 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] regionserver.HRegion(1676): Region close journal for a5436f89d60238ad7387432d34167717: Waiting for close lock at 1732137324945Running coprocessor pre-close hooks at 1732137324945Disabling compacts and flushes for region at 1732137324945Disabling writes for close at 1732137324945Obtaining lock to block concurrent updates at 1732137324945Preparing flush snapshotting stores in a5436f89d60238ad7387432d34167717 at 1732137324945Finished memstore snapshotting TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732137324953 (+8 ms)Flushing stores of TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717. at 1732137324954 (+1 ms)Flushing a5436f89d60238ad7387432d34167717/A: creating writer at 1732137324954Flushing a5436f89d60238ad7387432d34167717/A: appending metadata at 1732137324958 (+4 ms)Flushing a5436f89d60238ad7387432d34167717/A: closing flushed file at 1732137324958Flushing a5436f89d60238ad7387432d34167717/B: creating writer at 1732137324977 (+19 ms)Flushing a5436f89d60238ad7387432d34167717/B: appending metadata at 1732137324986 (+9 ms)Flushing a5436f89d60238ad7387432d34167717/B: closing flushed file at 1732137324986Flushing a5436f89d60238ad7387432d34167717/C: creating writer at 1732137325404 (+418 ms)Flushing a5436f89d60238ad7387432d34167717/C: appending metadata at 1732137325408 (+4 ms)Flushing a5436f89d60238ad7387432d34167717/C: closing flushed file at 1732137325408Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5bf5a606: reopening flushed file at 1732137325835 (+427 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64735c10: reopening flushed file at 1732137325840 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e35cfe2: reopening flushed file at 1732137325844 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for a5436f89d60238ad7387432d34167717 in 905ms, sequenceid=547, compaction requested=true at 1732137325850 (+6 ms)Writing region close event to WAL at 1732137325971 (+121 ms)Running coprocessor post-close hooks at 1732137325973 (+2 ms)Closed at 1732137325974 (+1 ms) 2024-11-20T21:15:25,975 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=81}] handler.UnassignRegionHandler(157): Closed a5436f89d60238ad7387432d34167717 2024-11-20T21:15:25,976 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=80 updating hbase:meta row=a5436f89d60238ad7387432d34167717, regionState=CLOSED 2024-11-20T21:15:25,977 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=81, ppid=80, state=RUNNABLE, hasLock=false; CloseRegionProcedure a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 because future has completed 2024-11-20T21:15:25,979 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-20T21:15:25,979 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; CloseRegionProcedure a5436f89d60238ad7387432d34167717, server=0b16ab3f13b7,36987,1732137236536 in 1.1870 sec 2024-11-20T21:15:25,981 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-11-20T21:15:25,981 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a5436f89d60238ad7387432d34167717, UNASSIGN in 1.1910 sec 2024-11-20T21:15:25,983 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-20T21:15:25,983 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1950 sec 2024-11-20T21:15:25,984 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732137325984"}]},"ts":"1732137325984"} 2024-11-20T21:15:25,985 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T21:15:25,985 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T21:15:25,987 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.2060 sec 2024-11-20T21:15:26,243 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-20T21:15:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-20T21:15:26,927 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:15:26,928 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-20T21:15:26,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:15:26,929 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=82, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:15:26,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-20T21:15:26,930 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=82, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:15:26,932 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717 2024-11-20T21:15:26,934 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A, FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B, FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C, FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/recovered.edits] 2024-11-20T21:15:26,936 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/4a39d7765b8148a8b65ec8776dff5c37 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/4a39d7765b8148a8b65ec8776dff5c37 2024-11-20T21:15:26,937 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/b7084acc33be44e38e8c4d5073b3d383 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/b7084acc33be44e38e8c4d5073b3d383 2024-11-20T21:15:26,938 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/f1646e9a74ee4a259efc97b3bfbadd02 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/A/f1646e9a74ee4a259efc97b3bfbadd02 2024-11-20T21:15:26,940 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/3cb1b72903e949bea406c534572449dc to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/3cb1b72903e949bea406c534572449dc 2024-11-20T21:15:26,941 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/8689378cabc9406b817b8ed211f77269 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/8689378cabc9406b817b8ed211f77269 2024-11-20T21:15:26,943 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/d8a4cf3dfa4c49bc9a8b8798aa74a6ac to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/B/d8a4cf3dfa4c49bc9a8b8798aa74a6ac 2024-11-20T21:15:26,945 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/158f19033ee44698bcd3d1657f40a1f2 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/158f19033ee44698bcd3d1657f40a1f2 2024-11-20T21:15:26,946 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/27b05e061a6e4768b0e01e366a3c04a5 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/27b05e061a6e4768b0e01e366a3c04a5 2024-11-20T21:15:26,947 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/a4854849cd634dd2b34f5b268d21cf0a to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/a4854849cd634dd2b34f5b268d21cf0a 2024-11-20T21:15:26,948 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/a93640f08324472e90ea2d04052502e6 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/C/a93640f08324472e90ea2d04052502e6 2024-11-20T21:15:26,950 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/recovered.edits/550.seqid to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717/recovered.edits/550.seqid 2024-11-20T21:15:26,951 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/a5436f89d60238ad7387432d34167717 2024-11-20T21:15:26,951 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T21:15:26,953 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=82, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:15:26,955 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T21:15:26,957 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T21:15:26,957 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=82, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:15:26,958 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T21:15:26,958 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732137326958"}]},"ts":"9223372036854775807"} 2024-11-20T21:15:26,959 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T21:15:26,959 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a5436f89d60238ad7387432d34167717, NAME => 'TestAcidGuarantees,,1732137299228.a5436f89d60238ad7387432d34167717.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T21:15:26,959 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T21:15:26,960 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732137326959"}]},"ts":"9223372036854775807"} 2024-11-20T21:15:26,961 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T21:15:26,962 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=82, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:15:26,963 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 34 msec 2024-11-20T21:15:27,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-20T21:15:27,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T21:15:27,037 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:15:27,046 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=247 (was 247), OpenFileDescriptor=453 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=833 (was 687) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=825 (was 1570) 2024-11-20T21:15:27,056 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=247, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=833, ProcessCount=11, AvailableMemoryMB=824 2024-11-20T21:15:27,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T21:15:27,057 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T21:15:27,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T21:15:27,059 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T21:15:27,059 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:27,059 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 83 2024-11-20T21:15:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-20T21:15:27,060 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T21:15:27,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742209_1385 (size=1061) 2024-11-20T21:15:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-20T21:15:27,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-20T21:15:27,467 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094 2024-11-20T21:15:27,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742210_1386 (size=53) 2024-11-20T21:15:27,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-20T21:15:27,874 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:15:27,874 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 67de1f67a4e12320e34a07470b17ec6e, disabling compactions & flushes 2024-11-20T21:15:27,874 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:27,874 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:27,874 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. after waiting 0 ms 2024-11-20T21:15:27,874 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:27,874 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:27,874 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 67de1f67a4e12320e34a07470b17ec6e: Waiting for close lock at 1732137327874Disabling compacts and flushes for region at 1732137327874Disabling writes for close at 1732137327874Writing region close event to WAL at 1732137327874Closed at 1732137327874 2024-11-20T21:15:27,875 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T21:15:27,876 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732137327875"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732137327875"}]},"ts":"1732137327875"} 2024-11-20T21:15:27,878 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T21:15:27,878 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T21:15:27,878 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732137327878"}]},"ts":"1732137327878"} 2024-11-20T21:15:27,880 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T21:15:27,880 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=67de1f67a4e12320e34a07470b17ec6e, ASSIGN}] 2024-11-20T21:15:27,881 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=67de1f67a4e12320e34a07470b17ec6e, ASSIGN 2024-11-20T21:15:27,882 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=67de1f67a4e12320e34a07470b17ec6e, ASSIGN; state=OFFLINE, location=0b16ab3f13b7,36987,1732137236536; forceNewPlan=false, retain=false 2024-11-20T21:15:28,033 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=67de1f67a4e12320e34a07470b17ec6e, regionState=OPENING, regionLocation=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:28,035 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=67de1f67a4e12320e34a07470b17ec6e, ASSIGN because future has completed 2024-11-20T21:15:28,036 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536}] 2024-11-20T21:15:28,192 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:28,192 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7752): Opening region: {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:15:28,193 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:28,193 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:15:28,193 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7794): checking encryption for 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:28,193 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(7797): checking classloading for 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:28,197 INFO [StoreOpener-67de1f67a4e12320e34a07470b17ec6e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:28,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-20T21:15:28,198 INFO [StoreOpener-67de1f67a4e12320e34a07470b17ec6e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T21:15:28,198 INFO [StoreOpener-67de1f67a4e12320e34a07470b17ec6e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 67de1f67a4e12320e34a07470b17ec6e columnFamilyName A 2024-11-20T21:15:28,198 DEBUG [StoreOpener-67de1f67a4e12320e34a07470b17ec6e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:28,199 INFO [StoreOpener-67de1f67a4e12320e34a07470b17ec6e-1 {}] regionserver.HStore(327): Store=67de1f67a4e12320e34a07470b17ec6e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:15:28,199 INFO [StoreOpener-67de1f67a4e12320e34a07470b17ec6e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:28,200 INFO [StoreOpener-67de1f67a4e12320e34a07470b17ec6e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T21:15:28,200 INFO [StoreOpener-67de1f67a4e12320e34a07470b17ec6e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 67de1f67a4e12320e34a07470b17ec6e columnFamilyName B 2024-11-20T21:15:28,200 DEBUG [StoreOpener-67de1f67a4e12320e34a07470b17ec6e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:28,201 INFO [StoreOpener-67de1f67a4e12320e34a07470b17ec6e-1 {}] regionserver.HStore(327): Store=67de1f67a4e12320e34a07470b17ec6e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:15:28,201 INFO [StoreOpener-67de1f67a4e12320e34a07470b17ec6e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:28,202 INFO [StoreOpener-67de1f67a4e12320e34a07470b17ec6e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T21:15:28,202 INFO [StoreOpener-67de1f67a4e12320e34a07470b17ec6e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 67de1f67a4e12320e34a07470b17ec6e columnFamilyName C 2024-11-20T21:15:28,202 DEBUG [StoreOpener-67de1f67a4e12320e34a07470b17ec6e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:28,203 INFO [StoreOpener-67de1f67a4e12320e34a07470b17ec6e-1 {}] regionserver.HStore(327): Store=67de1f67a4e12320e34a07470b17ec6e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:15:28,203 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:28,203 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1038): replaying wal for 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:28,204 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:28,204 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:28,205 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1048): stopping wal replay for 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:28,205 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1060): Cleaning up temporary data for 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:28,205 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T21:15:28,206 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1093): writing seq id for 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:28,208 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T21:15:28,209 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1114): Opened 67de1f67a4e12320e34a07470b17ec6e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67877899, jitterRate=0.011459514498710632}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T21:15:28,209 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:28,210 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegion(1006): Region open journal for 67de1f67a4e12320e34a07470b17ec6e: Running coprocessor pre-open hook at 1732137328193Writing region info on filesystem at 1732137328193Initializing all the Stores at 1732137328194 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732137328194Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732137328196 (+2 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732137328196Cleaning up temporary data from old regions at 1732137328205 (+9 ms)Running coprocessor post-open hooks at 1732137328209 (+4 ms)Region opened successfully at 1732137328210 (+1 ms) 2024-11-20T21:15:28,211 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., pid=85, masterSystemTime=1732137328189 2024-11-20T21:15:28,213 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:28,213 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=85}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:28,213 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=67de1f67a4e12320e34a07470b17ec6e, regionState=OPEN, openSeqNum=2, regionLocation=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:28,215 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; OpenRegionProcedure 67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 because future has completed 2024-11-20T21:15:28,217 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-20T21:15:28,218 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; OpenRegionProcedure 67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 in 180 msec 2024-11-20T21:15:28,219 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-20T21:15:28,219 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=67de1f67a4e12320e34a07470b17ec6e, ASSIGN in 338 msec 2024-11-20T21:15:28,220 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T21:15:28,220 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732137328220"}]},"ts":"1732137328220"} 2024-11-20T21:15:28,221 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T21:15:28,222 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=83, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T21:15:28,223 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1650 sec 2024-11-20T21:15:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-20T21:15:29,217 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:15:29,218 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30cfb621, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:29,219 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:15:29,219 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:15:29,221 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:15:29,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:15:29,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:15:29,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bb6fd7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:29,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:15:29,223 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:15:29,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:29,224 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:15:29,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a0a3785, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:29,226 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:15:29,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:15:29,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:15:29,228 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38276, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:15:29,229 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T21:15:29,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 0b16ab3f13b7,39687,1732137235812 2024-11-20T21:15:29,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5090b3bb 2024-11-20T21:15:29,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T21:15:29,231 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T21:15:29,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T21:15:29,233 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T21:15:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-20T21:15:29,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742211_1387 (size=1097) 2024-11-20T21:15:29,276 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-20T21:15:29,276 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-20T21:15:29,287 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T21:15:29,290 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=67de1f67a4e12320e34a07470b17ec6e, REOPEN/MOVE}] 2024-11-20T21:15:29,291 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=67de1f67a4e12320e34a07470b17ec6e, REOPEN/MOVE 2024-11-20T21:15:29,292 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=67de1f67a4e12320e34a07470b17ec6e, regionState=CLOSING, regionLocation=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:29,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=67de1f67a4e12320e34a07470b17ec6e, REOPEN/MOVE because future has completed 2024-11-20T21:15:29,296 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T21:15:29,296 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536}] 2024-11-20T21:15:29,448 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:29,448 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T21:15:29,449 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing 67de1f67a4e12320e34a07470b17ec6e, disabling compactions & flushes 2024-11-20T21:15:29,449 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:29,449 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:29,449 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. after waiting 0 ms 2024-11-20T21:15:29,449 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:29,452 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T21:15:29,453 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:29,453 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for 67de1f67a4e12320e34a07470b17ec6e: Waiting for close lock at 1732137329448Running coprocessor pre-close hooks at 1732137329448Disabling compacts and flushes for region at 1732137329448Disabling writes for close at 1732137329449 (+1 ms)Writing region close event to WAL at 1732137329449Running coprocessor post-close hooks at 1732137329453 (+4 ms)Closed at 1732137329453 2024-11-20T21:15:29,453 WARN [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionServer(3299): Not adding moved region record: 67de1f67a4e12320e34a07470b17ec6e to self. 2024-11-20T21:15:29,455 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:29,455 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=67de1f67a4e12320e34a07470b17ec6e, regionState=CLOSED 2024-11-20T21:15:29,457 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 because future has completed 2024-11-20T21:15:29,459 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-20T21:15:29,459 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure 67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 in 161 msec 2024-11-20T21:15:29,459 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=67de1f67a4e12320e34a07470b17ec6e, REOPEN/MOVE; state=CLOSED, location=0b16ab3f13b7,36987,1732137236536; forceNewPlan=false, retain=true 2024-11-20T21:15:29,610 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=67de1f67a4e12320e34a07470b17ec6e, regionState=OPENING, regionLocation=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:29,611 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=67de1f67a4e12320e34a07470b17ec6e, REOPEN/MOVE because future has completed 2024-11-20T21:15:29,612 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536}] 2024-11-20T21:15:29,766 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:29,766 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7752): Opening region: {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:15:29,766 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:29,766 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:15:29,766 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7794): checking encryption for 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:29,766 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(7797): checking classloading for 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:29,767 INFO [StoreOpener-67de1f67a4e12320e34a07470b17ec6e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:29,768 INFO [StoreOpener-67de1f67a4e12320e34a07470b17ec6e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T21:15:29,768 INFO [StoreOpener-67de1f67a4e12320e34a07470b17ec6e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 67de1f67a4e12320e34a07470b17ec6e columnFamilyName A 2024-11-20T21:15:29,769 DEBUG [StoreOpener-67de1f67a4e12320e34a07470b17ec6e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:29,770 INFO [StoreOpener-67de1f67a4e12320e34a07470b17ec6e-1 {}] regionserver.HStore(327): Store=67de1f67a4e12320e34a07470b17ec6e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:15:29,770 INFO [StoreOpener-67de1f67a4e12320e34a07470b17ec6e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:29,771 INFO [StoreOpener-67de1f67a4e12320e34a07470b17ec6e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T21:15:29,771 INFO [StoreOpener-67de1f67a4e12320e34a07470b17ec6e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 67de1f67a4e12320e34a07470b17ec6e columnFamilyName B 2024-11-20T21:15:29,771 DEBUG [StoreOpener-67de1f67a4e12320e34a07470b17ec6e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:29,772 INFO [StoreOpener-67de1f67a4e12320e34a07470b17ec6e-1 {}] regionserver.HStore(327): Store=67de1f67a4e12320e34a07470b17ec6e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:15:29,772 INFO [StoreOpener-67de1f67a4e12320e34a07470b17ec6e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:29,772 INFO [StoreOpener-67de1f67a4e12320e34a07470b17ec6e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T21:15:29,772 INFO [StoreOpener-67de1f67a4e12320e34a07470b17ec6e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 67de1f67a4e12320e34a07470b17ec6e columnFamilyName C 2024-11-20T21:15:29,772 DEBUG [StoreOpener-67de1f67a4e12320e34a07470b17ec6e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:29,773 INFO [StoreOpener-67de1f67a4e12320e34a07470b17ec6e-1 {}] regionserver.HStore(327): Store=67de1f67a4e12320e34a07470b17ec6e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:15:29,773 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:29,773 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1038): replaying wal for 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:29,774 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:29,775 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:29,776 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1048): stopping wal replay for 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:29,776 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1060): Cleaning up temporary data for 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:29,776 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T21:15:29,777 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1093): writing seq id for 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:29,778 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1114): Opened 67de1f67a4e12320e34a07470b17ec6e; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67136375, jitterRate=4.0994584560394287E-4}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T21:15:29,778 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:29,779 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegion(1006): Region open journal for 67de1f67a4e12320e34a07470b17ec6e: Running coprocessor pre-open hook at 1732137329766Writing region info on filesystem at 1732137329766Initializing all the Stores at 1732137329767 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732137329767Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732137329767Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732137329767Cleaning up temporary data from old regions at 1732137329776 (+9 ms)Running coprocessor post-open hooks at 1732137329778 (+2 ms)Region opened successfully at 1732137329779 (+1 ms) 2024-11-20T21:15:29,780 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., pid=90, masterSystemTime=1732137329763 2024-11-20T21:15:29,782 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:29,782 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=90}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:29,785 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=67de1f67a4e12320e34a07470b17ec6e, regionState=OPEN, openSeqNum=5, regionLocation=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:29,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 because future has completed 2024-11-20T21:15:29,788 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=88 2024-11-20T21:15:29,788 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 in 175 msec 2024-11-20T21:15:29,790 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-20T21:15:29,790 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=67de1f67a4e12320e34a07470b17ec6e, REOPEN/MOVE in 498 msec 2024-11-20T21:15:29,793 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-20T21:15:29,793 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 504 msec 2024-11-20T21:15:29,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 560 msec 2024-11-20T21:15:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-20T21:15:29,797 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:15:29,801 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67f5e8d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:29,801 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:15:29,801 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:15:29,802 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:15:29,802 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:15:29,802 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:15:29,802 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2792f786, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:29,802 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:15:29,803 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:15:29,803 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:29,804 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:15:29,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61e72bf3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:29,807 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ff38b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:29,807 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:15:29,807 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:15:29,808 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:15:29,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:15:29,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:15:29,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38a8b3f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:29,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:15:29,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:15:29,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:29,810 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:15:29,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e16b21b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:29,812 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ec6412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:29,812 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:15:29,812 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:15:29,813 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:15:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:15:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:15:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ea32ebc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:15:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:15:29,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:29,814 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:15:29,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2797f23e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:29,825 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f2378c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:29,825 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:15:29,825 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:15:29,826 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:15:29,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:15:29,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:15:29,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a8b7f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:29,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:15:29,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:15:29,826 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:29,827 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56930, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:15:29,828 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2340b131, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:29,832 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cc49f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:29,833 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:15:29,833 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:15:29,833 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:15:29,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:15:29,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:15:29,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d199b6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:29,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:15:29,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:15:29,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:29,835 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56936, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:15:29,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a4d4603, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:29,837 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f004f7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:29,837 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:15:29,837 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:15:29,838 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:15:29,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:15:29,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:15:29,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fc7c319, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:29,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:15:29,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:15:29,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:29,839 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:15:29,839 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55af244a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:29,840 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65616af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:29,840 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:15:29,840 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:15:29,841 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:15:29,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:15:29,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:15:29,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22f271be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:29,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:15:29,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:15:29,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:29,843 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56976, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:15:29,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45eca5c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:29,853 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74481d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:29,853 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:15:29,853 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:15:29,854 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:15:29,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:15:29,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:15:29,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@697e7fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:29,854 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:15:29,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:15:29,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:29,856 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:15:29,857 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ffe1b4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:29,869 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514b3d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:29,869 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:15:29,869 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:15:29,869 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:15:29,870 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:15:29,870 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:15:29,870 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22cd3155, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:29,870 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:15:29,870 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:15:29,870 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:29,871 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:15:29,872 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a66fddc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:29,877 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b14e3fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:29,877 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:15:29,877 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:15:29,878 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:15:29,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:15:29,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:15:29,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@138f2478, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:29,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:15:29,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:15:29,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:29,879 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57036, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:15:29,880 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ca19143, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:29,881 DEBUG [Thread-1737 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:15:29,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:15:29,882 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:15:29,883 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38284, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:15:29,885 DEBUG [Thread-1741 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:15:29,886 DEBUG [Thread-1750 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:15:29,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:15:29,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5] 2024-11-20T21:15:29,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:15:29,886 DEBUG [Thread-1754 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:15:29,886 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:15:29,887 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:15:29,887 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38288, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:15:29,887 DEBUG [Thread-1746 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:15:29,891 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5] 2024-11-20T21:15:29,897 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:15:29,897 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:15:29,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:15:29,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:15:29,897 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:15:29,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:15:29,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-20T21:15:29,898 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38300, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:15:29,899 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:15:29,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-20T21:15:29,900 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:15:29,900 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:15:29,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5] 2024-11-20T21:15:29,900 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:15:29,902 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5] 2024-11-20T21:15:29,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5] 2024-11-20T21:15:29,909 DEBUG [Thread-1748 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:15:29,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:29,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 67de1f67a4e12320e34a07470b17ec6e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T21:15:29,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=A 2024-11-20T21:15:29,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:29,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=B 2024-11-20T21:15:29,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:29,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=C 2024-11-20T21:15:29,909 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:15:29,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:29,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:15:29,911 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:15:29,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5] 2024-11-20T21:15:29,919 DEBUG [Thread-1752 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:15:29,921 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:15:29,922 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:15:29,923 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:15:29,927 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5] 2024-11-20T21:15:29,940 DEBUG [Thread-1735 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:15:29,940 DEBUG [Thread-1735 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:15:29,941 DEBUG [Thread-1735 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:15:29,941 DEBUG [Thread-1739 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:15:29,942 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38336, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:15:29,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:15:29,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:15:29,943 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38342, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:15:29,944 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5] 2024-11-20T21:15:29,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5] 2024-11-20T21:15:29,949 DEBUG [Thread-1743 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:15:29,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:15:29,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:15:29,958 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:15:29,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5] 2024-11-20T21:15:29,976 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f70ad28cccff444d9465811ea56641ca_67de1f67a4e12320e34a07470b17ec6e is 50, key is test_row_0/A:col10/1732137329906/Put/seqid=0 2024-11-20T21:15:29,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742212_1388 (size=12154) 2024-11-20T21:15:29,985 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:29,991 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f70ad28cccff444d9465811ea56641ca_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f70ad28cccff444d9465811ea56641ca_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:29,994 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/768acde1c2614807a349c3d0c8a7e9dd, store: [table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:29,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/768acde1c2614807a349c3d0c8a7e9dd is 175, key is test_row_0/A:col10/1732137329906/Put/seqid=0 2024-11-20T21:15:30,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-20T21:15:30,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:30,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137389994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:30,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:30,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:30,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137389994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:30,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137389996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:30,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:30,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:30,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732137329991, max timestamp=1732137330015Num uniques 859; 2024-11-20T21:15:30,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T21:15:30,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732137329991, max timestamp=1732137330015Num uniques 859; 2024-11-20T21:15:30,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:30,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T21:15:30,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732137329991, max timestamp=1732137330015Num uniques 859; 2024-11-20T21:15:30,023 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T21:15:30,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742213_1389 (size=30955) 2024-11-20T21:15:30,032 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/768acde1c2614807a349c3d0c8a7e9dd 2024-11-20T21:15:30,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-20T21:15:30,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:30,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:30,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:30,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:30,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:30,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:30,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/c5bcba740de64ca9b2f7418d46b8abb6 is 50, key is test_row_0/B:col10/1732137329906/Put/seqid=0 2024-11-20T21:15:30,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742214_1390 (size=12001) 2024-11-20T21:15:30,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/c5bcba740de64ca9b2f7418d46b8abb6 2024-11-20T21:15:30,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/013ebcf77ba64a7180e5af1aeae21afe is 50, key is test_row_0/C:col10/1732137329906/Put/seqid=0 2024-11-20T21:15:30,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742215_1391 (size=12001) 2024-11-20T21:15:30,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732137330090, max timestamp=1732137330148Num uniques 1018; 2024-11-20T21:15:30,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-20T21:15:30,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/013ebcf77ba64a7180e5af1aeae21afe 2024-11-20T21:15:30,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732137330090, max timestamp=1732137330148Num uniques 1018; 2024-11-20T21:15:30,162 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-20T21:15:30,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732137330090, max timestamp=1732137330148Num uniques 1018; 2024-11-20T21:15:30,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-20T21:15:30,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/768acde1c2614807a349c3d0c8a7e9dd as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/768acde1c2614807a349c3d0c8a7e9dd 2024-11-20T21:15:30,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-20T21:15:30,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:30,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:30,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:30,210 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:30,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:30,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:30,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-20T21:15:30,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/768acde1c2614807a349c3d0c8a7e9dd, entries=150, sequenceid=14, filesize=30.2 K 2024-11-20T21:15:30,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/c5bcba740de64ca9b2f7418d46b8abb6 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/c5bcba740de64ca9b2f7418d46b8abb6 2024-11-20T21:15:30,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:30,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137390225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:30,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:30,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137390226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:30,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:30,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:30,251 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:30,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137390229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:30,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,252 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:30,270 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/c5bcba740de64ca9b2f7418d46b8abb6, entries=150, sequenceid=14, filesize=11.7 K 2024-11-20T21:15:30,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:30,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137390250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:30,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:30,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137390251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:30,271 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:30,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,272 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:30,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/013ebcf77ba64a7180e5af1aeae21afe as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/013ebcf77ba64a7180e5af1aeae21afe 2024-11-20T21:15:30,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/013ebcf77ba64a7180e5af1aeae21afe, entries=150, sequenceid=14, filesize=11.7 K 2024-11-20T21:15:30,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 67de1f67a4e12320e34a07470b17ec6e in 384ms, sequenceid=14, compaction requested=false 2024-11-20T21:15:30,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:30,376 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 67de1f67a4e12320e34a07470b17ec6e 3/3 column families, dataSize=147.60 KB heapSize=409.92 KB 2024-11-20T21:15:30,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:30,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=A 2024-11-20T21:15:30,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:30,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=B 2024-11-20T21:15:30,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:30,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=C 2024-11-20T21:15:30,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:30,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-20T21:15:30,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:30,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:30,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:30,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:30,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:30,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:30,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120818929234563477896266e45c07bc25c_67de1f67a4e12320e34a07470b17ec6e is 50, key is test_row_0/A:col10/1732137330374/Put/seqid=0 2024-11-20T21:15:30,422 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:30,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137390400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:30,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,422 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,423 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:30,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:30,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137390407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:30,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:30,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:30,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:30,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137390417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:30,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,442 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:30,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:30,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137390420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:30,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,444 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137390409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:30,445 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:30,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:30,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742216_1392 (size=17034) 2024-11-20T21:15:30,457 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:30,492 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120818929234563477896266e45c07bc25c_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120818929234563477896266e45c07bc25c_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:30,493 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/b337c51257e64dbabd76e7b164fb8a14, store: [table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:30,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/b337c51257e64dbabd76e7b164fb8a14 is 175, key is test_row_0/A:col10/1732137330374/Put/seqid=0 2024-11-20T21:15:30,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-20T21:15:30,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742217_1393 (size=48139) 2024-11-20T21:15:30,529 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=68, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/b337c51257e64dbabd76e7b164fb8a14 2024-11-20T21:15:30,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:30,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137390527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:30,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:30,538 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/c9a882036699472da7528dcf0e609dcb is 50, key is test_row_0/B:col10/1732137330374/Put/seqid=0 2024-11-20T21:15:30,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-20T21:15:30,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:30,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:30,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:30,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] handler.RSProcedureHandler(58): pid=92 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:30,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=92 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:30,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=92 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:30,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742218_1394 (size=12001) 2024-11-20T21:15:30,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/c9a882036699472da7528dcf0e609dcb 2024-11-20T21:15:30,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:30,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137390547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:30,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:30,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137390547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:30,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,555 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:30,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:30,555 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:30,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137390548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:30,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:30,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137390548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:30,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:30,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:30,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:30,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/3ca4540d44d144c280b833ebdc8989f8 is 50, key is test_row_0/C:col10/1732137330374/Put/seqid=0 2024-11-20T21:15:30,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742219_1395 (size=12001) 2024-11-20T21:15:30,627 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/3ca4540d44d144c280b833ebdc8989f8 2024-11-20T21:15:30,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/b337c51257e64dbabd76e7b164fb8a14 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/b337c51257e64dbabd76e7b164fb8a14 2024-11-20T21:15:30,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/b337c51257e64dbabd76e7b164fb8a14, entries=250, sequenceid=68, filesize=47.0 K 2024-11-20T21:15:30,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/c9a882036699472da7528dcf0e609dcb as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/c9a882036699472da7528dcf0e609dcb 2024-11-20T21:15:30,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/c9a882036699472da7528dcf0e609dcb, entries=150, sequenceid=68, filesize=11.7 K 2024-11-20T21:15:30,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/3ca4540d44d144c280b833ebdc8989f8 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/3ca4540d44d144c280b833ebdc8989f8 2024-11-20T21:15:30,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/3ca4540d44d144c280b833ebdc8989f8, entries=150, sequenceid=68, filesize=11.7 K 2024-11-20T21:15:30,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~429.80 KB/440112, currentSize=40.25 KB/41220 for 67de1f67a4e12320e34a07470b17ec6e in 296ms, sequenceid=68, compaction requested=false 2024-11-20T21:15:30,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:30,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-20T21:15:30,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:30,696 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 67de1f67a4e12320e34a07470b17ec6e 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-20T21:15:30,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=A 2024-11-20T21:15:30,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:30,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=B 2024-11-20T21:15:30,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:30,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=C 2024-11-20T21:15:30,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:30,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205eb663fc687c4c42adf2f820d63c192a_67de1f67a4e12320e34a07470b17ec6e is 50, key is test_row_0/A:col10/1732137330407/Put/seqid=0 2024-11-20T21:15:30,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:30,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:30,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742220_1396 (size=12154) 2024-11-20T21:15:30,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732137330825, max timestamp=1732137330847Num uniques 859; 2024-11-20T21:15:30,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732137330825, max timestamp=1732137330847Num uniques 859; 2024-11-20T21:15:30,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T21:15:30,851 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T21:15:30,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732137330825, max timestamp=1732137330850Num uniques 859; 2024-11-20T21:15:30,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T21:15:30,966 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732137330905, max timestamp=1732137330956Num uniques 1018; 2024-11-20T21:15:30,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-20T21:15:30,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732137330905, max timestamp=1732137330956Num uniques 1018; 2024-11-20T21:15:30,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-20T21:15:30,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732137330905, max timestamp=1732137330956Num uniques 1018; 2024-11-20T21:15:30,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-20T21:15:31,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-20T21:15:31,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137391023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:31,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:31,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137391025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:31,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:31,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,052 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:31,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137391035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:31,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:31,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137391041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:31,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:31,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,061 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:31,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:31,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137391051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:31,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:31,168 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205eb663fc687c4c42adf2f820d63c192a_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205eb663fc687c4c42adf2f820d63c192a_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:31,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/80620780a16d4204b9e302f8e82d5a48, store: [table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:31,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/80620780a16d4204b9e302f8e82d5a48 is 175, key is test_row_0/A:col10/1732137330407/Put/seqid=0 2024-11-20T21:15:31,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:31,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137391157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:31,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:31,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:31,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137391158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:31,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:31,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:31,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137391167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:31,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:31,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137391167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:31,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:31,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:31,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:31,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137391176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:31,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:31,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742221_1397 (size=30955) 2024-11-20T21:15:31,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137391382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:31,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137391385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:31,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:31,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:31,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:31,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137391390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:31,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:31,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:31,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137391399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:31,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:31,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:31,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137391401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:31,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:31,513 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T21:15:31,609 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/80620780a16d4204b9e302f8e82d5a48 2024-11-20T21:15:31,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/b6f235f9d02048308f088b9ca11aa4d3 is 50, key is test_row_0/B:col10/1732137330407/Put/seqid=0 2024-11-20T21:15:31,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742222_1398 (size=12001) 2024-11-20T21:15:31,666 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/b6f235f9d02048308f088b9ca11aa4d3 2024-11-20T21:15:31,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/cc272099b4054d789b9bb01b0621001f is 50, key is test_row_0/C:col10/1732137330407/Put/seqid=0 2024-11-20T21:15:31,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742223_1399 (size=12001) 2024-11-20T21:15:31,691 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/cc272099b4054d789b9bb01b0621001f 2024-11-20T21:15:31,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/80620780a16d4204b9e302f8e82d5a48 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/80620780a16d4204b9e302f8e82d5a48 2024-11-20T21:15:31,712 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/80620780a16d4204b9e302f8e82d5a48, entries=150, sequenceid=77, filesize=30.2 K 2024-11-20T21:15:31,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:31,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137391698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:31,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:31,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:31,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137391706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:31,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:31,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/b6f235f9d02048308f088b9ca11aa4d3 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/b6f235f9d02048308f088b9ca11aa4d3 2024-11-20T21:15:31,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:31,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137391707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:31,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,723 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/b6f235f9d02048308f088b9ca11aa4d3, entries=150, sequenceid=77, filesize=11.7 K 2024-11-20T21:15:31,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:31,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/cc272099b4054d789b9bb01b0621001f as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/cc272099b4054d789b9bb01b0621001f 2024-11-20T21:15:31,727 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/cc272099b4054d789b9bb01b0621001f, entries=150, sequenceid=77, filesize=11.7 K 2024-11-20T21:15:31,728 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 67de1f67a4e12320e34a07470b17ec6e in 1032ms, sequenceid=77, compaction requested=true 2024-11-20T21:15:31,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:31,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:31,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-20T21:15:31,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-20T21:15:31,733 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-20T21:15:31,734 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8320 sec 2024-11-20T21:15:31,736 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 1.8370 sec 2024-11-20T21:15:31,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:31,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 67de1f67a4e12320e34a07470b17ec6e 3/3 column families, dataSize=147.60 KB heapSize=420 KB 2024-11-20T21:15:31,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=A 2024-11-20T21:15:31,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T21:15:31,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=B 2024-11-20T21:15:31,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T21:15:31,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=C 2024-11-20T21:15:31,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T21:15:31,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200b5c1e57ac59499fbbff41255206b30f_67de1f67a4e12320e34a07470b17ec6e is 50, key is test_row_0/A:col10/1732137331049/Put/seqid=0 2024-11-20T21:15:31,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742224_1400 (size=14744) 2024-11-20T21:15:31,774 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:31,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:31,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137391767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:31,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:31,780 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200b5c1e57ac59499fbbff41255206b30f_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200b5c1e57ac59499fbbff41255206b30f_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:31,781 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/b0ff3800e51043fb83ab23563e894830, store: [table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:31,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/b0ff3800e51043fb83ab23563e894830 is 175, key is test_row_0/A:col10/1732137331049/Put/seqid=0 2024-11-20T21:15:31,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:31,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137391775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:31,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,784 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:31,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742225_1401 (size=39699) 2024-11-20T21:15:31,788 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/b0ff3800e51043fb83ab23563e894830 2024-11-20T21:15:31,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/75d6344b34bf4bdfb0c00267488c1a92 is 50, key is test_row_0/B:col10/1732137331049/Put/seqid=0 2024-11-20T21:15:31,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742226_1402 (size=12101) 2024-11-20T21:15:31,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:31,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137391887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:31,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,892 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:31,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:31,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137391887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:31,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:31,903 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:32,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-20T21:15:32,047 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:15:32,048 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:15:32,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-20T21:15:32,050 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:15:32,051 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:15:32,051 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:15:32,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-20T21:15:32,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:32,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137392097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:32,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:32,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:32,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:32,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:32,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137392106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:32,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:32,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:32,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:32,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-20T21:15:32,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-20T21:15:32,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:32,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:32,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:32,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:32,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:32,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:32,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/75d6344b34bf4bdfb0c00267488c1a92 2024-11-20T21:15:32,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:32,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137392217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:32,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/5272c4dc64664a2b86f50e4c85f9ec7a is 50, key is test_row_0/C:col10/1732137331049/Put/seqid=0 2024-11-20T21:15:32,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:32,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137392227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:32,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:32,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:32,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:32,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:32,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137392228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:32,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:32,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:32,246 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:32,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742227_1403 (size=12101) 2024-11-20T21:15:32,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-20T21:15:32,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:32,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:32,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:32,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:32,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:32,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:32,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-20T21:15:32,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:32,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137392420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:32,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:32,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:32,433 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:32,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:32,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137392427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:32,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:32,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-20T21:15:32,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:32,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:32,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:32,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:32,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:32,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:32,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-20T21:15:32,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:32,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:32,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:32,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:32,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:32,671 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/5272c4dc64664a2b86f50e4c85f9ec7a 2024-11-20T21:15:32,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:32,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/b0ff3800e51043fb83ab23563e894830 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/b0ff3800e51043fb83ab23563e894830 2024-11-20T21:15:32,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-20T21:15:32,711 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/b0ff3800e51043fb83ab23563e894830, entries=200, sequenceid=132, filesize=38.8 K 2024-11-20T21:15:32,712 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/75d6344b34bf4bdfb0c00267488c1a92 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/75d6344b34bf4bdfb0c00267488c1a92 2024-11-20T21:15:32,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/75d6344b34bf4bdfb0c00267488c1a92, entries=150, sequenceid=132, filesize=11.8 K 2024-11-20T21:15:32,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/5272c4dc64664a2b86f50e4c85f9ec7a as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/5272c4dc64664a2b86f50e4c85f9ec7a 2024-11-20T21:15:32,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/5272c4dc64664a2b86f50e4c85f9ec7a, entries=150, sequenceid=132, filesize=11.8 K 2024-11-20T21:15:32,742 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~419.95 KB/430032, currentSize=33.54 KB/34350 for 67de1f67a4e12320e34a07470b17ec6e in 1006ms, sequenceid=132, compaction requested=true 2024-11-20T21:15:32,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:32,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 67de1f67a4e12320e34a07470b17ec6e:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:15:32,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:32,742 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:15:32,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 67de1f67a4e12320e34a07470b17ec6e:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:15:32,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:32,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 67de1f67a4e12320e34a07470b17ec6e:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:15:32,742 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T21:15:32,742 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:15:32,745 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149748 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:15:32,745 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 67de1f67a4e12320e34a07470b17ec6e/A is initiating minor compaction (all files) 2024-11-20T21:15:32,745 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 67de1f67a4e12320e34a07470b17ec6e/A in TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:32,745 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/768acde1c2614807a349c3d0c8a7e9dd, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/b337c51257e64dbabd76e7b164fb8a14, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/80620780a16d4204b9e302f8e82d5a48, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/b0ff3800e51043fb83ab23563e894830] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp, totalSize=146.2 K 2024-11-20T21:15:32,745 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:32,745 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. files: [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/768acde1c2614807a349c3d0c8a7e9dd, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/b337c51257e64dbabd76e7b164fb8a14, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/80620780a16d4204b9e302f8e82d5a48, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/b0ff3800e51043fb83ab23563e894830] 2024-11-20T21:15:32,748 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48104 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:15:32,748 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 768acde1c2614807a349c3d0c8a7e9dd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732137329898 2024-11-20T21:15:32,749 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 67de1f67a4e12320e34a07470b17ec6e/B is initiating minor compaction (all files) 2024-11-20T21:15:32,749 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 67de1f67a4e12320e34a07470b17ec6e/B in TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:32,749 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/c5bcba740de64ca9b2f7418d46b8abb6, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/c9a882036699472da7528dcf0e609dcb, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/b6f235f9d02048308f088b9ca11aa4d3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/75d6344b34bf4bdfb0c00267488c1a92] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp, totalSize=47.0 K 2024-11-20T21:15:32,749 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting b337c51257e64dbabd76e7b164fb8a14, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1732137330225 2024-11-20T21:15:32,749 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting c5bcba740de64ca9b2f7418d46b8abb6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732137329898 2024-11-20T21:15:32,750 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 80620780a16d4204b9e302f8e82d5a48, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732137330379 2024-11-20T21:15:32,750 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting c9a882036699472da7528dcf0e609dcb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1732137330225 2024-11-20T21:15:32,752 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0ff3800e51043fb83ab23563e894830, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732137331032 2024-11-20T21:15:32,752 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting b6f235f9d02048308f088b9ca11aa4d3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732137330379 2024-11-20T21:15:32,753 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 75d6344b34bf4bdfb0c00267488c1a92, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732137331032 2024-11-20T21:15:32,766 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:32,777 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 67de1f67a4e12320e34a07470b17ec6e#B#compaction#345 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:32,778 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/edbcf43ee1aa4c1ca4ad632659dba5c0 is 50, key is test_row_0/B:col10/1732137331049/Put/seqid=0 2024-11-20T21:15:32,782 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112075a04fa4d73b473f941b9ced4770e1cb_67de1f67a4e12320e34a07470b17ec6e store=[table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:32,785 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112075a04fa4d73b473f941b9ced4770e1cb_67de1f67a4e12320e34a07470b17ec6e, store=[table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:32,785 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112075a04fa4d73b473f941b9ced4770e1cb_67de1f67a4e12320e34a07470b17ec6e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:32,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-20T21:15:32,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:32,829 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 67de1f67a4e12320e34a07470b17ec6e 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-20T21:15:32,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=A 2024-11-20T21:15:32,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:32,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=B 2024-11-20T21:15:32,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:32,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=C 2024-11-20T21:15:32,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:32,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742228_1404 (size=12239) 2024-11-20T21:15:32,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742229_1405 (size=4469) 2024-11-20T21:15:32,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d07156e655d54f43bf5fdad83e85fbf8_67de1f67a4e12320e34a07470b17ec6e is 50, key is test_row_0/A:col10/1732137331737/Put/seqid=0 2024-11-20T21:15:32,864 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 67de1f67a4e12320e34a07470b17ec6e#A#compaction#344 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:32,864 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/d2336b8e0da3455382f9386c1765db15 is 175, key is test_row_0/A:col10/1732137331049/Put/seqid=0 2024-11-20T21:15:32,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742231_1407 (size=31193) 2024-11-20T21:15:32,908 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/d2336b8e0da3455382f9386c1765db15 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/d2336b8e0da3455382f9386c1765db15 2024-11-20T21:15:32,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742230_1406 (size=12304) 2024-11-20T21:15:32,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:32,916 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 67de1f67a4e12320e34a07470b17ec6e/A of 67de1f67a4e12320e34a07470b17ec6e into d2336b8e0da3455382f9386c1765db15(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:32,916 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:32,916 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., storeName=67de1f67a4e12320e34a07470b17ec6e/A, priority=12, startTime=1732137332742; duration=0sec 2024-11-20T21:15:32,916 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:32,916 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 67de1f67a4e12320e34a07470b17ec6e:A 2024-11-20T21:15:32,916 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:15:32,919 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120d07156e655d54f43bf5fdad83e85fbf8_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d07156e655d54f43bf5fdad83e85fbf8_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:32,920 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48104 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:15:32,920 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 67de1f67a4e12320e34a07470b17ec6e/C is initiating minor compaction (all files) 2024-11-20T21:15:32,920 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 67de1f67a4e12320e34a07470b17ec6e/C in TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:32,921 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/013ebcf77ba64a7180e5af1aeae21afe, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/3ca4540d44d144c280b833ebdc8989f8, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/cc272099b4054d789b9bb01b0621001f, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/5272c4dc64664a2b86f50e4c85f9ec7a] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp, totalSize=47.0 K 2024-11-20T21:15:32,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/f80232103adf4203a10fdd523b01f93c, store: [table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:32,921 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 013ebcf77ba64a7180e5af1aeae21afe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732137329898 2024-11-20T21:15:32,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/f80232103adf4203a10fdd523b01f93c is 175, key is test_row_0/A:col10/1732137331737/Put/seqid=0 2024-11-20T21:15:32,923 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ca4540d44d144c280b833ebdc8989f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1732137330225 2024-11-20T21:15:32,924 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc272099b4054d789b9bb01b0621001f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732137330379 2024-11-20T21:15:32,924 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5272c4dc64664a2b86f50e4c85f9ec7a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732137331032 2024-11-20T21:15:32,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742232_1408 (size=31105) 2024-11-20T21:15:32,939 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=140, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/f80232103adf4203a10fdd523b01f93c 2024-11-20T21:15:32,943 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 67de1f67a4e12320e34a07470b17ec6e#C#compaction#347 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:32,944 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/d213b70e63504a8795f410877a640873 is 50, key is test_row_0/C:col10/1732137331049/Put/seqid=0 2024-11-20T21:15:32,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:32,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:32,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/cd2d93259717447fa1fe8171c49f2d57 is 50, key is test_row_0/B:col10/1732137331737/Put/seqid=0 2024-11-20T21:15:32,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742234_1410 (size=12151) 2024-11-20T21:15:32,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742233_1409 (size=12239) 2024-11-20T21:15:33,092 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732137333008, max timestamp=1732137333087Num uniques 859; 2024-11-20T21:15:33,092 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732137333008, max timestamp=1732137333087Num uniques 859; 2024-11-20T21:15:33,092 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T21:15:33,092 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732137333008, max timestamp=1732137333087Num uniques 859; 2024-11-20T21:15:33,092 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T21:15:33,092 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T21:15:33,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-20T21:15:33,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732137333172, max timestamp=1732137333242Num uniques 1068; 2024-11-20T21:15:33,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-20T21:15:33,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732137333209, max timestamp=1732137333242Num uniques 1068; 2024-11-20T21:15:33,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-20T21:15:33,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732137333209, max timestamp=1732137333242Num uniques 1068; 2024-11-20T21:15:33,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-20T21:15:33,265 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/edbcf43ee1aa4c1ca4ad632659dba5c0 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/edbcf43ee1aa4c1ca4ad632659dba5c0 2024-11-20T21:15:33,271 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 67de1f67a4e12320e34a07470b17ec6e/B of 67de1f67a4e12320e34a07470b17ec6e into edbcf43ee1aa4c1ca4ad632659dba5c0(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:33,271 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:33,272 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., storeName=67de1f67a4e12320e34a07470b17ec6e/B, priority=12, startTime=1732137332742; duration=0sec 2024-11-20T21:15:33,272 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:33,272 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 67de1f67a4e12320e34a07470b17ec6e:B 2024-11-20T21:15:33,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732137333306, max timestamp=1732137333329Num uniques 1077; 2024-11-20T21:15:33,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-20T21:15:33,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732137333306, max timestamp=1732137333329Num uniques 1027; 2024-11-20T21:15:33,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-20T21:15:33,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732137333306, max timestamp=1732137333329Num uniques 1027; 2024-11-20T21:15:33,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-20T21:15:33,367 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/cd2d93259717447fa1fe8171c49f2d57 2024-11-20T21:15:33,389 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/d213b70e63504a8795f410877a640873 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/d213b70e63504a8795f410877a640873 2024-11-20T21:15:33,398 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 67de1f67a4e12320e34a07470b17ec6e/C of 67de1f67a4e12320e34a07470b17ec6e into d213b70e63504a8795f410877a640873(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:33,398 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:33,398 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., storeName=67de1f67a4e12320e34a07470b17ec6e/C, priority=12, startTime=1732137332742; duration=0sec 2024-11-20T21:15:33,398 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:33,398 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 67de1f67a4e12320e34a07470b17ec6e:C 2024-11-20T21:15:33,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/3630c1db28334e63a17b25a443d5fb03 is 50, key is test_row_0/C:col10/1732137331737/Put/seqid=0 2024-11-20T21:15:33,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732137333361, max timestamp=1732137333419Num uniques 1036; 2024-11-20T21:15:33,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-20T21:15:33,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732137333361, max timestamp=1732137333419Num uniques 1036; 2024-11-20T21:15:33,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-20T21:15:33,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1732137333361, max timestamp=1732137333419Num uniques 1036; 2024-11-20T21:15:33,428 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47438, new segment data size=9520, suffix heap size=135712,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=208 2024-11-20T21:15:33,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742235_1411 (size=12151) 2024-11-20T21:15:33,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1732137333462, max timestamp=1732137333518Num uniques 1095; 2024-11-20T21:15:33,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=15800, suffix heap size=141848,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=345 2024-11-20T21:15:33,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15800, totalHeapSize=36120, min timestamp=1732137333462, max timestamp=1732137333518Num uniques 1095; 2024-11-20T21:15:33,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=15800, suffix heap size=141848,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=345 2024-11-20T21:15:33,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=345, cellSize=15790, totalHeapSize=36120, min timestamp=1732137333462, max timestamp=1732137333518Num uniques 1067; 2024-11-20T21:15:33,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=15790, suffix heap size=138936,new segment heap size=36120 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=345 2024-11-20T21:15:33,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:33,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137393565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:33,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:33,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:33,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:33,587 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:33,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137393574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:33,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:33,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:33,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:33,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:33,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137393584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:33,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:33,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:33,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:33,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:33,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137393585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:33,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:33,602 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:33,602 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:33,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137393586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:33,602 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:33,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:33,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:33,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:33,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:33,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137393701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:33,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:33,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:33,708 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:33,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:33,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137393701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:33,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137393707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:33,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137393707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:33,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137393708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:33,710 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:33,833 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/3630c1db28334e63a17b25a443d5fb03 2024-11-20T21:15:33,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/f80232103adf4203a10fdd523b01f93c as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/f80232103adf4203a10fdd523b01f93c 2024-11-20T21:15:33,847 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/f80232103adf4203a10fdd523b01f93c, entries=150, sequenceid=140, filesize=30.4 K 2024-11-20T21:15:33,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/cd2d93259717447fa1fe8171c49f2d57 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/cd2d93259717447fa1fe8171c49f2d57 2024-11-20T21:15:33,867 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/cd2d93259717447fa1fe8171c49f2d57, entries=150, sequenceid=140, filesize=11.9 K 2024-11-20T21:15:33,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/3630c1db28334e63a17b25a443d5fb03 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/3630c1db28334e63a17b25a443d5fb03 2024-11-20T21:15:33,877 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/3630c1db28334e63a17b25a443d5fb03, entries=150, sequenceid=140, filesize=11.9 K 2024-11-20T21:15:33,881 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=154.31 KB/158010 for 67de1f67a4e12320e34a07470b17ec6e in 1052ms, sequenceid=140, compaction requested=false 2024-11-20T21:15:33,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:33,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:33,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-20T21:15:33,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-20T21:15:33,887 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-20T21:15:33,887 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8340 sec 2024-11-20T21:15:33,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 1.8400 sec 2024-11-20T21:15:33,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:33,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 67de1f67a4e12320e34a07470b17ec6e 3/3 column families, dataSize=161.02 KB heapSize=447.23 KB 2024-11-20T21:15:33,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=A 2024-11-20T21:15:33,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:33,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=B 2024-11-20T21:15:33,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:33,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=C 2024-11-20T21:15:33,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:33,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b275b351be7b4f0b8c591ff48c49b946_67de1f67a4e12320e34a07470b17ec6e is 50, key is test_row_0/A:col10/1732137333585/Put/seqid=0 2024-11-20T21:15:33,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:33,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137393930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:33,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:33,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137393933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:33,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:33,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:33,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137393934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:33,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:33,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:33,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:33,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:33,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:33,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:33,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:33,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:33,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:33,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137393937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:33,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:33,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:33,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:33,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:33,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137393937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:33,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:33,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:33,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:33,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742236_1412 (size=14794) 2024-11-20T21:15:34,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:34,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137394057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:34,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:34,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137394057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:34,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:34,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:34,066 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:34,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:34,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:34,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:34,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137394057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:34,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:34,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:34,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:34,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137394057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:34,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:34,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:34,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:34,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:34,067 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:34,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:34,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137394067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:34,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:34,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:34,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:34,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-20T21:15:34,207 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:15:34,207 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:15:34,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-20T21:15:34,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-20T21:15:34,209 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:15:34,210 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:15:34,210 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:15:34,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:34,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137394276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:34,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:34,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:34,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:34,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:34,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137394276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:34,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:34,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:34,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137394277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:34,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:34,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:34,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:34,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:34,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137394277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:34,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:34,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:34,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:34,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:34,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:34,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137394277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:34,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:34,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:34,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:34,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:34,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-20T21:15:34,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-20T21:15:34,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:34,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:34,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:34,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:34,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:34,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:34,375 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:34,379 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b275b351be7b4f0b8c591ff48c49b946_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b275b351be7b4f0b8c591ff48c49b946_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:34,380 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/4cfdcc552c2446ac8c1779993852430e, store: [table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:34,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/4cfdcc552c2446ac8c1779993852430e is 175, key is test_row_0/A:col10/1732137333585/Put/seqid=0 2024-11-20T21:15:34,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742237_1413 (size=39749) 2024-11-20T21:15:34,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-20T21:15:34,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:34,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:34,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:34,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:34,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:34,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:34,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-20T21:15:34,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137394597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:34,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:34,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:34,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:34,606 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137394597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:34,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:34,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:34,606 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:34,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137394597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:34,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:34,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:34,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:34,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137394597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:34,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:34,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:34,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:34,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137394597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:34,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:34,607 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:34,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:34,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:34,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:34,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-20T21:15:34,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:34,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:34,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:34,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:34,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:34,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:34,789 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/4cfdcc552c2446ac8c1779993852430e 2024-11-20T21:15:34,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/12cecd95b61846fc9d2ba6b94583d97d is 50, key is test_row_0/B:col10/1732137333585/Put/seqid=0 2024-11-20T21:15:34,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742238_1414 (size=12151) 2024-11-20T21:15:34,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/12cecd95b61846fc9d2ba6b94583d97d 2024-11-20T21:15:34,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-20T21:15:34,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:34,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:34,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:34,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:34,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-20T21:15:34,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/585a3d207633462696ea685d5df25528 is 50, key is test_row_0/C:col10/1732137333585/Put/seqid=0 2024-11-20T21:15:34,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742239_1415 (size=12151) 2024-11-20T21:15:34,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-20T21:15:34,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:34,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:34,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:34,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:34,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:34,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:35,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:35,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137395117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:35,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:35,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:35,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:35,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:35,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137395118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:35,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:35,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:35,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:35,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137395118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:35,136 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:35,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:35,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137395118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:35,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:35,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:35,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:35,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:35,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137395118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:35,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:35,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:35,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:35,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:35,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:35,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:35,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-20T21:15:35,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:35,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:35,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:35,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:35,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:35,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:35,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/585a3d207633462696ea685d5df25528 2024-11-20T21:15:35,296 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/4cfdcc552c2446ac8c1779993852430e as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/4cfdcc552c2446ac8c1779993852430e 2024-11-20T21:15:35,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-20T21:15:35,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:35,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:35,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:35,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:35,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:35,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:35,329 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/4cfdcc552c2446ac8c1779993852430e, entries=200, sequenceid=249, filesize=38.8 K 2024-11-20T21:15:35,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/12cecd95b61846fc9d2ba6b94583d97d as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/12cecd95b61846fc9d2ba6b94583d97d 2024-11-20T21:15:35,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/12cecd95b61846fc9d2ba6b94583d97d, entries=150, sequenceid=249, filesize=11.9 K 2024-11-20T21:15:35,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/585a3d207633462696ea685d5df25528 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/585a3d207633462696ea685d5df25528 2024-11-20T21:15:35,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-20T21:15:35,352 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/585a3d207633462696ea685d5df25528, entries=150, sequenceid=249, filesize=11.9 K 2024-11-20T21:15:35,353 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~447.19 KB/457920, currentSize=26.84 KB/27480 for 67de1f67a4e12320e34a07470b17ec6e in 1424ms, sequenceid=249, compaction requested=true 2024-11-20T21:15:35,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:35,353 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:35,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 67de1f67a4e12320e34a07470b17ec6e:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:15:35,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:35,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 67de1f67a4e12320e34a07470b17ec6e:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:15:35,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:35,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 67de1f67a4e12320e34a07470b17ec6e:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:15:35,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:35,353 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:35,354 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:35,354 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 67de1f67a4e12320e34a07470b17ec6e/A is initiating minor compaction (all files) 2024-11-20T21:15:35,354 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 67de1f67a4e12320e34a07470b17ec6e/A in TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:35,354 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/d2336b8e0da3455382f9386c1765db15, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/f80232103adf4203a10fdd523b01f93c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/4cfdcc552c2446ac8c1779993852430e] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp, totalSize=99.7 K 2024-11-20T21:15:35,354 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:35,354 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. files: [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/d2336b8e0da3455382f9386c1765db15, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/f80232103adf4203a10fdd523b01f93c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/4cfdcc552c2446ac8c1779993852430e] 2024-11-20T21:15:35,356 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36541 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:35,356 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 67de1f67a4e12320e34a07470b17ec6e/B is initiating minor compaction (all files) 2024-11-20T21:15:35,356 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 67de1f67a4e12320e34a07470b17ec6e/B in TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:35,356 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/edbcf43ee1aa4c1ca4ad632659dba5c0, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/cd2d93259717447fa1fe8171c49f2d57, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/12cecd95b61846fc9d2ba6b94583d97d] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp, totalSize=35.7 K 2024-11-20T21:15:35,360 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2336b8e0da3455382f9386c1765db15, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732137331032 2024-11-20T21:15:35,362 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f80232103adf4203a10fdd523b01f93c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732137331737 2024-11-20T21:15:35,362 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cfdcc552c2446ac8c1779993852430e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732137333561 2024-11-20T21:15:35,362 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting edbcf43ee1aa4c1ca4ad632659dba5c0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732137331032 2024-11-20T21:15:35,364 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting cd2d93259717447fa1fe8171c49f2d57, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732137331737 2024-11-20T21:15:35,364 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 12cecd95b61846fc9d2ba6b94583d97d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732137333582 2024-11-20T21:15:35,369 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:35,381 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411203193044cbec34566bc5ae8578fdc4245_67de1f67a4e12320e34a07470b17ec6e store=[table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:35,383 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411203193044cbec34566bc5ae8578fdc4245_67de1f67a4e12320e34a07470b17ec6e, store=[table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:35,383 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411203193044cbec34566bc5ae8578fdc4245_67de1f67a4e12320e34a07470b17ec6e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:35,394 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 67de1f67a4e12320e34a07470b17ec6e#B#compaction#354 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:35,394 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/c4d32ad2df2a4d5cb191242c5cb0df14 is 50, key is test_row_0/B:col10/1732137333585/Put/seqid=0 2024-11-20T21:15:35,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742240_1416 (size=4469) 2024-11-20T21:15:35,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742241_1417 (size=12391) 2024-11-20T21:15:35,429 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 67de1f67a4e12320e34a07470b17ec6e#A#compaction#353 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:35,429 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/2c385662618a4bb0ab60dde891b47716 is 175, key is test_row_0/A:col10/1732137333585/Put/seqid=0 2024-11-20T21:15:35,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742242_1418 (size=31345) 2024-11-20T21:15:35,459 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/c4d32ad2df2a4d5cb191242c5cb0df14 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/c4d32ad2df2a4d5cb191242c5cb0df14 2024-11-20T21:15:35,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-20T21:15:35,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:35,462 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 67de1f67a4e12320e34a07470b17ec6e 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-20T21:15:35,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=A 2024-11-20T21:15:35,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:35,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=B 2024-11-20T21:15:35,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:35,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=C 2024-11-20T21:15:35,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:35,469 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 67de1f67a4e12320e34a07470b17ec6e/B of 67de1f67a4e12320e34a07470b17ec6e into c4d32ad2df2a4d5cb191242c5cb0df14(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:35,469 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:35,469 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., storeName=67de1f67a4e12320e34a07470b17ec6e/B, priority=13, startTime=1732137335353; duration=0sec 2024-11-20T21:15:35,469 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:35,469 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 67de1f67a4e12320e34a07470b17ec6e:B 2024-11-20T21:15:35,469 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:35,470 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/2c385662618a4bb0ab60dde891b47716 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/2c385662618a4bb0ab60dde891b47716 2024-11-20T21:15:35,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112066340f11376244eba3f3d42c4a79ae78_67de1f67a4e12320e34a07470b17ec6e is 50, key is test_row_0/A:col10/1732137333932/Put/seqid=0 2024-11-20T21:15:35,478 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36541 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:35,478 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 67de1f67a4e12320e34a07470b17ec6e/C is initiating minor compaction (all files) 2024-11-20T21:15:35,478 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 67de1f67a4e12320e34a07470b17ec6e/C in TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:35,478 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/d213b70e63504a8795f410877a640873, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/3630c1db28334e63a17b25a443d5fb03, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/585a3d207633462696ea685d5df25528] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp, totalSize=35.7 K 2024-11-20T21:15:35,479 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting d213b70e63504a8795f410877a640873, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1732137331032 2024-11-20T21:15:35,479 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 3630c1db28334e63a17b25a443d5fb03, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732137331737 2024-11-20T21:15:35,479 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 585a3d207633462696ea685d5df25528, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732137333582 2024-11-20T21:15:35,481 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 67de1f67a4e12320e34a07470b17ec6e/A of 67de1f67a4e12320e34a07470b17ec6e into 2c385662618a4bb0ab60dde891b47716(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:35,481 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:35,481 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., storeName=67de1f67a4e12320e34a07470b17ec6e/A, priority=13, startTime=1732137335353; duration=0sec 2024-11-20T21:15:35,481 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:35,481 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 67de1f67a4e12320e34a07470b17ec6e:A 2024-11-20T21:15:35,498 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 67de1f67a4e12320e34a07470b17ec6e#C#compaction#356 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:35,499 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/19737f98c2784931ac944c85a11aee57 is 50, key is test_row_0/C:col10/1732137333585/Put/seqid=0 2024-11-20T21:15:35,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742243_1419 (size=12304) 2024-11-20T21:15:35,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742244_1420 (size=12391) 2024-11-20T21:15:35,523 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/19737f98c2784931ac944c85a11aee57 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/19737f98c2784931ac944c85a11aee57 2024-11-20T21:15:35,530 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 67de1f67a4e12320e34a07470b17ec6e/C of 67de1f67a4e12320e34a07470b17ec6e into 19737f98c2784931ac944c85a11aee57(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:35,531 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:35,531 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., storeName=67de1f67a4e12320e34a07470b17ec6e/C, priority=13, startTime=1732137335353; duration=0sec 2024-11-20T21:15:35,531 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:35,531 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 67de1f67a4e12320e34a07470b17ec6e:C 2024-11-20T21:15:35,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:35,935 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112066340f11376244eba3f3d42c4a79ae78_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112066340f11376244eba3f3d42c4a79ae78_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:35,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/8e0ba7e180a0457ea9e37c6b89becfcf, store: [table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:35,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/8e0ba7e180a0457ea9e37c6b89becfcf is 175, key is test_row_0/A:col10/1732137333932/Put/seqid=0 2024-11-20T21:15:35,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742245_1421 (size=31105) 2024-11-20T21:15:36,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:36,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:36,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732137336173, max timestamp=1732137336207Num uniques 859; 2024-11-20T21:15:36,213 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T21:15:36,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732137336173, max timestamp=1732137336207Num uniques 859; 2024-11-20T21:15:36,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T21:15:36,216 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732137336173, max timestamp=1732137336207Num uniques 859; 2024-11-20T21:15:36,216 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T21:15:36,244 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-20T21:15:36,244 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-20T21:15:36,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732137336263, max timestamp=1732137336299Num uniques 1118; 2024-11-20T21:15:36,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-20T21:15:36,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732137336263, max timestamp=1732137336299Num uniques 1118; 2024-11-20T21:15:36,306 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-20T21:15:36,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732137336263, max timestamp=1732137336299Num uniques 1118; 2024-11-20T21:15:36,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-20T21:15:36,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-20T21:15:36,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732137336346, max timestamp=1732137336367Num uniques 1077; 2024-11-20T21:15:36,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732137336346, max timestamp=1732137336367Num uniques 1077; 2024-11-20T21:15:36,374 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-20T21:15:36,373 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-20T21:15:36,376 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1732137336346, max timestamp=1732137336367Num uniques 1077; 2024-11-20T21:15:36,376 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-20T21:15:36,390 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=256, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/8e0ba7e180a0457ea9e37c6b89becfcf 2024-11-20T21:15:36,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/4cd298e128ac464aba96ddbc164c4953 is 50, key is test_row_0/B:col10/1732137333932/Put/seqid=0 2024-11-20T21:15:36,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742246_1422 (size=12151) 2024-11-20T21:15:36,442 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1732137336413, max timestamp=1732137336437Num uniques 1086; 2024-11-20T21:15:36,442 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=15386, suffix heap size=140912,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=336 2024-11-20T21:15:36,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1732137336413, max timestamp=1732137336437Num uniques 1086; 2024-11-20T21:15:36,446 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=15386, suffix heap size=140912,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=336 2024-11-20T21:15:36,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1732137336413, max timestamp=1732137336437Num uniques 1086; 2024-11-20T21:15:36,450 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=15386, suffix heap size=140912,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=336 2024-11-20T21:15:36,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137396503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:36,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:36,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:36,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:36,524 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137396506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:36,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:36,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732137336487, max timestamp=1732137336518Num uniques 1195; 2024-11-20T21:15:36,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:36,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:36,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:36,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=11220, suffix heap size=152248,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=245 2024-11-20T21:15:36,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137396508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:36,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732137336487, max timestamp=1732137336518Num uniques 1195; 2024-11-20T21:15:36,525 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=11220, suffix heap size=152248,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=245 2024-11-20T21:15:36,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:36,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:36,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:36,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732137336487, max timestamp=1732137336518Num uniques 1195; 2024-11-20T21:15:36,528 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=11220, suffix heap size=152248,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=245 2024-11-20T21:15:36,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137396637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:36,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:36,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:36,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:36,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:36,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137396646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:36,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:36,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:36,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:36,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732137336581, max timestamp=1732137336650Num uniques 1104; 2024-11-20T21:15:36,656 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-20T21:15:36,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732137336581, max timestamp=1732137336650Num uniques 1104; 2024-11-20T21:15:36,657 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-20T21:15:36,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732137336581, max timestamp=1732137336650Num uniques 1104; 2024-11-20T21:15:36,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-20T21:15:36,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732137336714, max timestamp=1732137336736Num uniques 1063; 2024-11-20T21:15:36,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-20T21:15:36,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732137336714, max timestamp=1732137336736Num uniques 1063; 2024-11-20T21:15:36,742 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-20T21:15:36,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1732137336714, max timestamp=1732137336736Num uniques 1063; 2024-11-20T21:15:36,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-20T21:15:36,810 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/4cd298e128ac464aba96ddbc164c4953 2024-11-20T21:15:36,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/080bf6471d0a4d788ea09836bf7bf129 is 50, key is test_row_0/C:col10/1732137333932/Put/seqid=0 2024-11-20T21:15:36,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742247_1423 (size=12151) 2024-11-20T21:15:36,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1732137336809, max timestamp=1732137336832Num uniques 1072; 2024-11-20T21:15:36,840 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-20T21:15:36,843 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1732137336809, max timestamp=1732137336832Num uniques 1072; 2024-11-20T21:15:36,844 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-20T21:15:36,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1732137336809, max timestamp=1732137336832Num uniques 1072; 2024-11-20T21:15:36,849 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-20T21:15:36,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:36,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137396910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:36,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:36,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:36,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:36,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137396911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:36,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:36,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:36,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:36,930 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:36,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:36,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137396929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:36,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1732137336907, max timestamp=1732137336926Num uniques 1081; 2024-11-20T21:15:36,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-20T21:15:36,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:36,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:36,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:36,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1732137336907, max timestamp=1732137336926Num uniques 1081; 2024-11-20T21:15:36,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-20T21:15:36,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732137336907, max timestamp=1732137336926Num uniques 1081; 2024-11-20T21:15:36,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=6870, suffix heap size=140392,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=150 2024-11-20T21:15:37,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1732137337022, max timestamp=1732137337048Num uniques 1040; 2024-11-20T21:15:37,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-11-20T21:15:37,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1732137337022, max timestamp=1732137337048Num uniques 1040; 2024-11-20T21:15:37,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-11-20T21:15:37,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1732137337022, max timestamp=1732137337048Num uniques 1009; 2024-11-20T21:15:37,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10990, suffix heap size=132904,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=240 2024-11-20T21:15:37,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1732137337092, max timestamp=1732137337120Num uniques 1099; 2024-11-20T21:15:37,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-20T21:15:37,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1732137337092, max timestamp=1732137337120Num uniques 1099; 2024-11-20T21:15:37,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-20T21:15:37,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1732137337092, max timestamp=1732137337120Num uniques 1099; 2024-11-20T21:15:37,134 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-20T21:15:37,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:37,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137397187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:37,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=358, cellSize=16390, totalHeapSize=37472, min timestamp=1732137337165, max timestamp=1732137337194Num uniques 1108; 2024-11-20T21:15:37,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=16390, suffix heap size=143200,new segment heap size=37472 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=358 2024-11-20T21:15:37,199 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=358, cellSize=16390, totalHeapSize=37472, min timestamp=1732137337165, max timestamp=1732137337194Num uniques 1108; 2024-11-20T21:15:37,200 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=16390, suffix heap size=143200,new segment heap size=37472 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=358 2024-11-20T21:15:37,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=358, cellSize=16390, totalHeapSize=37472, min timestamp=1732137337165, max timestamp=1732137337194Num uniques 1108; 2024-11-20T21:15:37,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=16390, suffix heap size=143200,new segment heap size=37472 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=358 2024-11-20T21:15:37,229 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/080bf6471d0a4d788ea09836bf7bf129 2024-11-20T21:15:37,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/8e0ba7e180a0457ea9e37c6b89becfcf as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/8e0ba7e180a0457ea9e37c6b89becfcf 2024-11-20T21:15:37,241 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/8e0ba7e180a0457ea9e37c6b89becfcf, entries=150, sequenceid=256, filesize=30.4 K 2024-11-20T21:15:37,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/4cd298e128ac464aba96ddbc164c4953 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/4cd298e128ac464aba96ddbc164c4953 2024-11-20T21:15:37,249 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/4cd298e128ac464aba96ddbc164c4953, entries=150, sequenceid=256, filesize=11.9 K 2024-11-20T21:15:37,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/080bf6471d0a4d788ea09836bf7bf129 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/080bf6471d0a4d788ea09836bf7bf129 2024-11-20T21:15:37,257 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/080bf6471d0a4d788ea09836bf7bf129, entries=150, sequenceid=256, filesize=11.9 K 2024-11-20T21:15:37,258 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=120.76 KB/123660 for 67de1f67a4e12320e34a07470b17ec6e in 1796ms, sequenceid=256, compaction requested=false 2024-11-20T21:15:37,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:37,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:37,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-20T21:15:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-20T21:15:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:37,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 67de1f67a4e12320e34a07470b17ec6e 3/3 column families, dataSize=127.47 KB heapSize=346.41 KB 2024-11-20T21:15:37,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=A 2024-11-20T21:15:37,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:37,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=B 2024-11-20T21:15:37,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:37,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=C 2024-11-20T21:15:37,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:37,266 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-20T21:15:37,266 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0530 sec 2024-11-20T21:15:37,270 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 3.0590 sec 2024-11-20T21:15:37,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201d88c164277d47f2bea8e7492165cfdc_67de1f67a4e12320e34a07470b17ec6e is 50, key is test_row_0/A:col10/1732137337260/Put/seqid=0 2024-11-20T21:15:37,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:37,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137397285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:37,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,301 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:37,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:37,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137397288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:37,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:37,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137397288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:37,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:37,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:37,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137397296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:37,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:37,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742248_1424 (size=14994) 2024-11-20T21:15:37,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:37,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137397308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:37,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:37,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:37,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137397407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:37,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:37,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137397408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:37,420 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:37,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:37,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137397408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:37,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:37,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:37,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137397408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:37,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:37,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:37,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:37,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137397527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:37,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:37,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:37,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137397629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:37,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:37,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137397629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:37,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:37,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:37,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:37,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137397629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:37,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:37,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:37,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137397630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:37,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:37,707 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:37,711 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411201d88c164277d47f2bea8e7492165cfdc_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201d88c164277d47f2bea8e7492165cfdc_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:37,711 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/101e8f648f0c42ea8fdaf27d54d78787, store: [table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:37,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/101e8f648f0c42ea8fdaf27d54d78787 is 175, key is test_row_0/A:col10/1732137337260/Put/seqid=0 2024-11-20T21:15:37,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742249_1425 (size=39949) 2024-11-20T21:15:37,716 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=481, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/101e8f648f0c42ea8fdaf27d54d78787 2024-11-20T21:15:37,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/f1a569c9b31743b4a3ca2f9cc3f14082 is 50, key is test_row_0/B:col10/1732137337260/Put/seqid=0 2024-11-20T21:15:37,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742250_1426 (size=12301) 2024-11-20T21:15:37,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:37,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137397847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:37,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:37,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137397947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:37,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137397947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:37,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,961 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:37,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:37,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137397947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:37,961 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:37,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:37,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137397948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:37,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,962 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:37,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:37,962 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:38,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/f1a569c9b31743b4a3ca2f9cc3f14082 2024-11-20T21:15:38,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/610158eb29c544199878a923bcf295f8 is 50, key is test_row_0/C:col10/1732137337260/Put/seqid=0 2024-11-20T21:15:38,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742251_1427 (size=12301) 2024-11-20T21:15:38,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:38,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137398367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:38,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:38,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:38,381 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:38,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-20T21:15:38,387 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:15:38,388 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:15:38,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-20T21:15:38,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-20T21:15:38,390 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:15:38,391 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:15:38,391 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:15:38,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:38,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137398467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:38,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:38,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137398467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:38,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:38,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:38,476 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:38,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:38,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:38,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:38,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137398467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:38,476 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:38,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:38,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137398467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:38,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:38,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:38,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:38,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:38,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:38,477 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:38,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-20T21:15:38,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-20T21:15:38,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:38,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:38,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:38,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:38,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:38,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:38,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=481 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/610158eb29c544199878a923bcf295f8 2024-11-20T21:15:38,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/101e8f648f0c42ea8fdaf27d54d78787 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/101e8f648f0c42ea8fdaf27d54d78787 2024-11-20T21:15:38,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/101e8f648f0c42ea8fdaf27d54d78787, entries=200, sequenceid=481, filesize=39.0 K 2024-11-20T21:15:38,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/f1a569c9b31743b4a3ca2f9cc3f14082 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/f1a569c9b31743b4a3ca2f9cc3f14082 2024-11-20T21:15:38,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/f1a569c9b31743b4a3ca2f9cc3f14082, entries=150, sequenceid=481, filesize=12.0 K 2024-11-20T21:15:38,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/610158eb29c544199878a923bcf295f8 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/610158eb29c544199878a923bcf295f8 2024-11-20T21:15:38,583 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/610158eb29c544199878a923bcf295f8, entries=150, sequenceid=481, filesize=12.0 K 2024-11-20T21:15:38,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~366.28 KB/375072, currentSize=53.67 KB/54960 for 67de1f67a4e12320e34a07470b17ec6e in 1324ms, sequenceid=481, compaction requested=true 2024-11-20T21:15:38,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:38,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 67de1f67a4e12320e34a07470b17ec6e:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:15:38,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:38,584 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:38,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 67de1f67a4e12320e34a07470b17ec6e:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:15:38,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:38,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 67de1f67a4e12320e34a07470b17ec6e:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:15:38,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:38,584 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:38,585 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:38,585 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 67de1f67a4e12320e34a07470b17ec6e/A is initiating minor compaction (all files) 2024-11-20T21:15:38,585 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 67de1f67a4e12320e34a07470b17ec6e/A in TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:38,585 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/2c385662618a4bb0ab60dde891b47716, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/8e0ba7e180a0457ea9e37c6b89becfcf, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/101e8f648f0c42ea8fdaf27d54d78787] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp, totalSize=100.0 K 2024-11-20T21:15:38,585 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:38,585 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. files: [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/2c385662618a4bb0ab60dde891b47716, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/8e0ba7e180a0457ea9e37c6b89becfcf, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/101e8f648f0c42ea8fdaf27d54d78787] 2024-11-20T21:15:38,586 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36843 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:38,586 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 67de1f67a4e12320e34a07470b17ec6e/B is initiating minor compaction (all files) 2024-11-20T21:15:38,586 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 67de1f67a4e12320e34a07470b17ec6e/B in TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:38,586 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c385662618a4bb0ab60dde891b47716, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732137333582 2024-11-20T21:15:38,586 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/c4d32ad2df2a4d5cb191242c5cb0df14, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/4cd298e128ac464aba96ddbc164c4953, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/f1a569c9b31743b4a3ca2f9cc3f14082] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp, totalSize=36.0 K 2024-11-20T21:15:38,586 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e0ba7e180a0457ea9e37c6b89becfcf, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732137333931 2024-11-20T21:15:38,586 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting c4d32ad2df2a4d5cb191242c5cb0df14, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732137333582 2024-11-20T21:15:38,586 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 101e8f648f0c42ea8fdaf27d54d78787, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732137337240 2024-11-20T21:15:38,587 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cd298e128ac464aba96ddbc164c4953, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732137333931 2024-11-20T21:15:38,587 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting f1a569c9b31743b4a3ca2f9cc3f14082, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732137337240 2024-11-20T21:15:38,593 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:38,599 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 67de1f67a4e12320e34a07470b17ec6e#B#compaction#363 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:38,599 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/3f6952793a524c648d56972e42696acf is 50, key is test_row_0/B:col10/1732137337260/Put/seqid=0 2024-11-20T21:15:38,604 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112044adbbe9a98c4af58844b2e70309d194_67de1f67a4e12320e34a07470b17ec6e store=[table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:38,605 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112044adbbe9a98c4af58844b2e70309d194_67de1f67a4e12320e34a07470b17ec6e, store=[table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:38,605 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112044adbbe9a98c4af58844b2e70309d194_67de1f67a4e12320e34a07470b17ec6e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:38,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742253_1429 (size=4469) 2024-11-20T21:15:38,614 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 67de1f67a4e12320e34a07470b17ec6e#A#compaction#362 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:38,614 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/89c270f02e944546ad67c08daaa5bc60 is 175, key is test_row_0/A:col10/1732137337260/Put/seqid=0 2024-11-20T21:15:38,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742252_1428 (size=12643) 2024-11-20T21:15:38,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742254_1430 (size=31597) 2024-11-20T21:15:38,627 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/3f6952793a524c648d56972e42696acf as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/3f6952793a524c648d56972e42696acf 2024-11-20T21:15:38,633 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 67de1f67a4e12320e34a07470b17ec6e/B of 67de1f67a4e12320e34a07470b17ec6e into 3f6952793a524c648d56972e42696acf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:38,633 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:38,633 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., storeName=67de1f67a4e12320e34a07470b17ec6e/B, priority=13, startTime=1732137338584; duration=0sec 2024-11-20T21:15:38,633 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:38,633 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 67de1f67a4e12320e34a07470b17ec6e:B 2024-11-20T21:15:38,633 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:38,634 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36843 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:38,634 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 67de1f67a4e12320e34a07470b17ec6e/C is initiating minor compaction (all files) 2024-11-20T21:15:38,635 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 67de1f67a4e12320e34a07470b17ec6e/C in TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:38,635 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/19737f98c2784931ac944c85a11aee57, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/080bf6471d0a4d788ea09836bf7bf129, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/610158eb29c544199878a923bcf295f8] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp, totalSize=36.0 K 2024-11-20T21:15:38,635 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 19737f98c2784931ac944c85a11aee57, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732137333582 2024-11-20T21:15:38,635 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 080bf6471d0a4d788ea09836bf7bf129, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732137333931 2024-11-20T21:15:38,636 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 610158eb29c544199878a923bcf295f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732137337240 2024-11-20T21:15:38,652 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 67de1f67a4e12320e34a07470b17ec6e#C#compaction#364 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:38,653 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/5185a0fb9fd44d8892e25cceb86a1384 is 50, key is test_row_0/C:col10/1732137337260/Put/seqid=0 2024-11-20T21:15:38,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742255_1431 (size=12643) 2024-11-20T21:15:38,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-20T21:15:38,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:38,700 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 67de1f67a4e12320e34a07470b17ec6e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T21:15:38,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=A 2024-11-20T21:15:38,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:38,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=B 2024-11-20T21:15:38,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:38,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=C 2024-11-20T21:15:38,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:38,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202dc1eeba990f4216843877a476ba08ae_67de1f67a4e12320e34a07470b17ec6e is 50, key is test_row_0/A:col10/1732137337294/Put/seqid=0 2024-11-20T21:15:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-20T21:15:38,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742256_1432 (size=12454) 2024-11-20T21:15:38,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:38,727 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202dc1eeba990f4216843877a476ba08ae_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202dc1eeba990f4216843877a476ba08ae_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:38,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/b7b6ae19ba254481bc4d6a44a0d8cbce, store: [table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:38,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/b7b6ae19ba254481bc4d6a44a0d8cbce is 175, key is test_row_0/A:col10/1732137337294/Put/seqid=0 2024-11-20T21:15:38,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742257_1433 (size=31255) 2024-11-20T21:15:38,745 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=493, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/b7b6ae19ba254481bc4d6a44a0d8cbce 2024-11-20T21:15:38,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/2dfb0dd397a84bb2972306cfcc2cef6f is 50, key is test_row_0/B:col10/1732137337294/Put/seqid=0 2024-11-20T21:15:38,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742258_1434 (size=12301) 2024-11-20T21:15:39,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-20T21:15:39,029 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/89c270f02e944546ad67c08daaa5bc60 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/89c270f02e944546ad67c08daaa5bc60 2024-11-20T21:15:39,033 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 67de1f67a4e12320e34a07470b17ec6e/A of 67de1f67a4e12320e34a07470b17ec6e into 89c270f02e944546ad67c08daaa5bc60(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:39,033 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:39,033 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., storeName=67de1f67a4e12320e34a07470b17ec6e/A, priority=13, startTime=1732137338584; duration=0sec 2024-11-20T21:15:39,034 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:39,034 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 67de1f67a4e12320e34a07470b17ec6e:A 2024-11-20T21:15:39,074 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/5185a0fb9fd44d8892e25cceb86a1384 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/5185a0fb9fd44d8892e25cceb86a1384 2024-11-20T21:15:39,078 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 67de1f67a4e12320e34a07470b17ec6e/C of 67de1f67a4e12320e34a07470b17ec6e into 5185a0fb9fd44d8892e25cceb86a1384(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:39,078 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:39,078 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., storeName=67de1f67a4e12320e34a07470b17ec6e/C, priority=13, startTime=1732137338584; duration=0sec 2024-11-20T21:15:39,078 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:39,078 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 67de1f67a4e12320e34a07470b17ec6e:C 2024-11-20T21:15:39,174 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/2dfb0dd397a84bb2972306cfcc2cef6f 2024-11-20T21:15:39,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/1cad4d43872048018d4881d844aa8a48 is 50, key is test_row_0/C:col10/1732137337294/Put/seqid=0 2024-11-20T21:15:39,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742259_1435 (size=12301) 2024-11-20T21:15:39,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:39,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:39,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732137339492, max timestamp=1732137339509Num uniques 859; 2024-11-20T21:15:39,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732137339492, max timestamp=1732137339509Num uniques 859; 2024-11-20T21:15:39,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T21:15:39,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T21:15:39,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732137339492, max timestamp=1732137339509Num uniques 859; 2024-11-20T21:15:39,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T21:15:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-20T21:15:39,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:39,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137399551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:39,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:39,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:39,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:39,569 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:39,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137399558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:39,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:39,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:39,570 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:39,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:39,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137399559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:39,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:39,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137399568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:39,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:39,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:39,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137399569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:39,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:39,618 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=493 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/1cad4d43872048018d4881d844aa8a48 2024-11-20T21:15:39,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/b7b6ae19ba254481bc4d6a44a0d8cbce as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/b7b6ae19ba254481bc4d6a44a0d8cbce 2024-11-20T21:15:39,628 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/b7b6ae19ba254481bc4d6a44a0d8cbce, entries=150, sequenceid=493, filesize=30.5 K 2024-11-20T21:15:39,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/2dfb0dd397a84bb2972306cfcc2cef6f as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/2dfb0dd397a84bb2972306cfcc2cef6f 2024-11-20T21:15:39,631 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/2dfb0dd397a84bb2972306cfcc2cef6f, entries=150, sequenceid=493, filesize=12.0 K 2024-11-20T21:15:39,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/1cad4d43872048018d4881d844aa8a48 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/1cad4d43872048018d4881d844aa8a48 2024-11-20T21:15:39,634 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/1cad4d43872048018d4881d844aa8a48, entries=150, sequenceid=493, filesize=12.0 K 2024-11-20T21:15:39,635 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 67de1f67a4e12320e34a07470b17ec6e in 935ms, sequenceid=493, compaction requested=false 2024-11-20T21:15:39,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:39,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:39,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-20T21:15:39,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-20T21:15:39,638 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-20T21:15:39,638 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2450 sec 2024-11-20T21:15:39,640 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.2510 sec 2024-11-20T21:15:39,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:39,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 67de1f67a4e12320e34a07470b17ec6e 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-20T21:15:39,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=A 2024-11-20T21:15:39,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:39,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=B 2024-11-20T21:15:39,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:39,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=C 2024-11-20T21:15:39,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:39,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120cf72a21e66664db4ac00f26b18e88a91_67de1f67a4e12320e34a07470b17ec6e is 50, key is test_row_0/A:col10/1732137339568/Put/seqid=0 2024-11-20T21:15:39,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:39,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137399698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:39,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:39,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:39,711 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:39,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742260_1436 (size=14994) 2024-11-20T21:15:39,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:39,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137399709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:39,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:39,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:39,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:39,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:39,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137399711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:39,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:39,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:39,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:39,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:39,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137399712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:39,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:39,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:39,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:39,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:39,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137399712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:39,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:39,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:39,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:39,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137399821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:39,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:39,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:39,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:39,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137399837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:39,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:39,855 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:39,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:39,856 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:39,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137399840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:39,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:39,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:39,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:39,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:39,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137399842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:39,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:39,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137399845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:39,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:39,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:39,858 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:39,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:39,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:39,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:40,063 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137400047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:40,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:40,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:40,064 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:40,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137400067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:40,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:40,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137400069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:40,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:40,087 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:40,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:40,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:40,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137400069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:40,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:40,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:40,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:40,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:40,087 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137400070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:40,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:40,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:40,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:40,118 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:40,129 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120cf72a21e66664db4ac00f26b18e88a91_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120cf72a21e66664db4ac00f26b18e88a91_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:40,129 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/ad889b4625fa405f8976bd264400540f, store: [table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:40,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/ad889b4625fa405f8976bd264400540f is 175, key is test_row_0/A:col10/1732137339568/Put/seqid=0 2024-11-20T21:15:40,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742261_1437 (size=39949) 2024-11-20T21:15:40,146 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=531, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/ad889b4625fa405f8976bd264400540f 2024-11-20T21:15:40,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/3703b4d49bb341f48801a8879b699176 is 50, key is test_row_0/B:col10/1732137339568/Put/seqid=0 2024-11-20T21:15:40,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742262_1438 (size=12301) 2024-11-20T21:15:40,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/3703b4d49bb341f48801a8879b699176 2024-11-20T21:15:40,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/e6c3f619702d4cf1bfa316785870a10e is 50, key is test_row_0/C:col10/1732137339568/Put/seqid=0 2024-11-20T21:15:40,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742263_1439 (size=12301) 2024-11-20T21:15:40,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:40,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137400377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:40,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:40,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:40,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:40,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:40,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137400397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:40,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:40,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137400397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:40,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:40,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137400397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:40,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:40,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:40,417 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:40,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:40,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137400398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:40,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:40,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:40,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:40,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:40,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:40,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:40,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-20T21:15:40,537 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:15:40,538 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:15:40,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-20T21:15:40,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T21:15:40,540 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:15:40,541 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:15:40,541 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:15:40,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=531 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/e6c3f619702d4cf1bfa316785870a10e 2024-11-20T21:15:40,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/ad889b4625fa405f8976bd264400540f as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/ad889b4625fa405f8976bd264400540f 2024-11-20T21:15:40,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/ad889b4625fa405f8976bd264400540f, entries=200, sequenceid=531, filesize=39.0 K 2024-11-20T21:15:40,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/3703b4d49bb341f48801a8879b699176 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/3703b4d49bb341f48801a8879b699176 2024-11-20T21:15:40,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/3703b4d49bb341f48801a8879b699176, entries=150, sequenceid=531, filesize=12.0 K 2024-11-20T21:15:40,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/e6c3f619702d4cf1bfa316785870a10e as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/e6c3f619702d4cf1bfa316785870a10e 2024-11-20T21:15:40,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T21:15:40,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/e6c3f619702d4cf1bfa316785870a10e, entries=150, sequenceid=531, filesize=12.0 K 2024-11-20T21:15:40,655 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for 67de1f67a4e12320e34a07470b17ec6e in 969ms, sequenceid=531, compaction requested=true 2024-11-20T21:15:40,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:40,656 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:40,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 67de1f67a4e12320e34a07470b17ec6e:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:15:40,656 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102801 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:40,657 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 67de1f67a4e12320e34a07470b17ec6e/A is initiating minor compaction (all files) 2024-11-20T21:15:40,657 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 67de1f67a4e12320e34a07470b17ec6e/A in TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:40,657 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/89c270f02e944546ad67c08daaa5bc60, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/b7b6ae19ba254481bc4d6a44a0d8cbce, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/ad889b4625fa405f8976bd264400540f] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp, totalSize=100.4 K 2024-11-20T21:15:40,657 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:40,657 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. files: [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/89c270f02e944546ad67c08daaa5bc60, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/b7b6ae19ba254481bc4d6a44a0d8cbce, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/ad889b4625fa405f8976bd264400540f] 2024-11-20T21:15:40,657 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89c270f02e944546ad67c08daaa5bc60, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732137337240 2024-11-20T21:15:40,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:40,657 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:40,657 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7b6ae19ba254481bc4d6a44a0d8cbce, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732137337283 2024-11-20T21:15:40,658 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad889b4625fa405f8976bd264400540f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1732137339556 2024-11-20T21:15:40,658 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:40,659 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 67de1f67a4e12320e34a07470b17ec6e/B is initiating minor compaction (all files) 2024-11-20T21:15:40,659 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 67de1f67a4e12320e34a07470b17ec6e/B in TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:40,659 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/3f6952793a524c648d56972e42696acf, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/2dfb0dd397a84bb2972306cfcc2cef6f, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/3703b4d49bb341f48801a8879b699176] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp, totalSize=36.4 K 2024-11-20T21:15:40,659 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f6952793a524c648d56972e42696acf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732137337240 2024-11-20T21:15:40,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 67de1f67a4e12320e34a07470b17ec6e:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:15:40,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:40,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 67de1f67a4e12320e34a07470b17ec6e:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:15:40,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:40,659 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dfb0dd397a84bb2972306cfcc2cef6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732137337283 2024-11-20T21:15:40,660 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 3703b4d49bb341f48801a8879b699176, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1732137339556 2024-11-20T21:15:40,673 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:40,675 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411202bed2fb26b4142d48e123565593bb986_67de1f67a4e12320e34a07470b17ec6e store=[table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:40,676 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 67de1f67a4e12320e34a07470b17ec6e#B#compaction#372 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:40,677 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/1db4b67f65d94d5fa871f2cf96dbd858 is 50, key is test_row_0/B:col10/1732137339568/Put/seqid=0 2024-11-20T21:15:40,685 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411202bed2fb26b4142d48e123565593bb986_67de1f67a4e12320e34a07470b17ec6e, store=[table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:40,685 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202bed2fb26b4142d48e123565593bb986_67de1f67a4e12320e34a07470b17ec6e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:40,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742264_1440 (size=12745) 2024-11-20T21:15:40,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-20T21:15:40,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:40,695 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 67de1f67a4e12320e34a07470b17ec6e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T21:15:40,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=A 2024-11-20T21:15:40,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:40,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=B 2024-11-20T21:15:40,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:40,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=C 2024-11-20T21:15:40,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:40,706 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/1db4b67f65d94d5fa871f2cf96dbd858 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/1db4b67f65d94d5fa871f2cf96dbd858 2024-11-20T21:15:40,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742265_1441 (size=4469) 2024-11-20T21:15:40,715 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 67de1f67a4e12320e34a07470b17ec6e/B of 67de1f67a4e12320e34a07470b17ec6e into 1db4b67f65d94d5fa871f2cf96dbd858(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:40,716 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:40,716 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., storeName=67de1f67a4e12320e34a07470b17ec6e/B, priority=13, startTime=1732137340657; duration=0sec 2024-11-20T21:15:40,716 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:40,716 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 67de1f67a4e12320e34a07470b17ec6e:B 2024-11-20T21:15:40,716 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:40,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e30f64eaca034b4ab25560246d3d17d2_67de1f67a4e12320e34a07470b17ec6e is 50, key is test_row_0/A:col10/1732137339709/Put/seqid=0 2024-11-20T21:15:40,721 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:40,721 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 67de1f67a4e12320e34a07470b17ec6e/C is initiating minor compaction (all files) 2024-11-20T21:15:40,721 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 67de1f67a4e12320e34a07470b17ec6e/C in TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:40,721 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/5185a0fb9fd44d8892e25cceb86a1384, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/1cad4d43872048018d4881d844aa8a48, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/e6c3f619702d4cf1bfa316785870a10e] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp, totalSize=36.4 K 2024-11-20T21:15:40,721 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 5185a0fb9fd44d8892e25cceb86a1384, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=481, earliestPutTs=1732137337240 2024-11-20T21:15:40,722 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cad4d43872048018d4881d844aa8a48, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=493, earliestPutTs=1732137337283 2024-11-20T21:15:40,722 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting e6c3f619702d4cf1bfa316785870a10e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1732137339556 2024-11-20T21:15:40,728 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 67de1f67a4e12320e34a07470b17ec6e#C#compaction#374 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:40,728 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/2b4382dd717c4485adbc57cf3f3ad2d2 is 50, key is test_row_0/C:col10/1732137339568/Put/seqid=0 2024-11-20T21:15:40,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742266_1442 (size=12454) 2024-11-20T21:15:40,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742267_1443 (size=12745) 2024-11-20T21:15:40,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T21:15:40,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:40,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:40,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:40,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137400980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:40,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:40,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:40,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:40,992 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732137340953, max timestamp=1732137340989Num uniques 859; 2024-11-20T21:15:40,992 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T21:15:40,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732137340953, max timestamp=1732137340989Num uniques 859; 2024-11-20T21:15:40,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T21:15:40,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732137340953, max timestamp=1732137340989Num uniques 859; 2024-11-20T21:15:40,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T21:15:41,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:41,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137401062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:41,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:41,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137401071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:41,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:41,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137401080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:41,094 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:41,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,094 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:41,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,094 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:41,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137401081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:41,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,094 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,095 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:41,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,096 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:41,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:41,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137401096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:41,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:41,120 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 67de1f67a4e12320e34a07470b17ec6e#A#compaction#371 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:41,120 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/ccacf18537da4821947bfdf245017eed is 175, key is test_row_0/A:col10/1732137339568/Put/seqid=0 2024-11-20T21:15:41,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:41,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742268_1444 (size=31699) 2024-11-20T21:15:41,139 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e30f64eaca034b4ab25560246d3d17d2_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e30f64eaca034b4ab25560246d3d17d2_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:41,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/0c5061cd15a74e589919bce268988fd2, store: [table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:41,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/0c5061cd15a74e589919bce268988fd2 is 175, key is test_row_0/A:col10/1732137339709/Put/seqid=0 2024-11-20T21:15:41,156 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/2b4382dd717c4485adbc57cf3f3ad2d2 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/2b4382dd717c4485adbc57cf3f3ad2d2 2024-11-20T21:15:41,159 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 67de1f67a4e12320e34a07470b17ec6e/C of 67de1f67a4e12320e34a07470b17ec6e into 2b4382dd717c4485adbc57cf3f3ad2d2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:41,160 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:41,160 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., storeName=67de1f67a4e12320e34a07470b17ec6e/C, priority=13, startTime=1732137340659; duration=0sec 2024-11-20T21:15:41,160 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:41,160 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 67de1f67a4e12320e34a07470b17ec6e:C 2024-11-20T21:15:41,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742269_1445 (size=31255) 2024-11-20T21:15:41,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T21:15:41,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:41,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137401197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:41,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:41,203 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:41,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137401197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:41,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:41,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,204 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:41,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137401197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:41,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,204 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:41,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:41,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137401206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,222 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:41,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:41,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137401317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:41,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:41,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:41,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137401407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:41,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:41,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137401407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:41,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:41,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:41,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137401407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:41,421 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:41,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:41,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:41,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137401426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:41,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:41,535 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/ccacf18537da4821947bfdf245017eed as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/ccacf18537da4821947bfdf245017eed 2024-11-20T21:15:41,540 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 67de1f67a4e12320e34a07470b17ec6e/A of 67de1f67a4e12320e34a07470b17ec6e into ccacf18537da4821947bfdf245017eed(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:41,540 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:41,540 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., storeName=67de1f67a4e12320e34a07470b17ec6e/A, priority=13, startTime=1732137340655; duration=0sec 2024-11-20T21:15:41,540 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:41,540 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 67de1f67a4e12320e34a07470b17ec6e:A 2024-11-20T21:15:41,563 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=542, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/0c5061cd15a74e589919bce268988fd2 2024-11-20T21:15:41,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/9382302d2c1b4381a0db8de3ea189d4f is 50, key is test_row_0/B:col10/1732137339709/Put/seqid=0 2024-11-20T21:15:41,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742270_1446 (size=12301) 2024-11-20T21:15:41,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:41,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137401636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:41,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:41,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T21:15:41,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:41,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137401727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:41,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:41,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:41,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137401727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:41,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:41,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137401729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:41,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:41,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:41,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:41,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137401746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:42,008 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=542 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/9382302d2c1b4381a0db8de3ea189d4f 2024-11-20T21:15:42,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/85f46fe9bc954be2810c857cc017ee81 is 50, key is test_row_0/C:col10/1732137339709/Put/seqid=0 2024-11-20T21:15:42,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742271_1447 (size=12301) 2024-11-20T21:15:42,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:42,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137402161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:42,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:42,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:42,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:42,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:42,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137402247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:42,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:42,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:42,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137402247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:42,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:42,256 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:42,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:42,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:42,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:42,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137402247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:42,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:42,256 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:42,256 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:42,256 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:42,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:42,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137402267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:42,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:42,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:42,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:42,484 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=542 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/85f46fe9bc954be2810c857cc017ee81 2024-11-20T21:15:42,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/0c5061cd15a74e589919bce268988fd2 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/0c5061cd15a74e589919bce268988fd2 2024-11-20T21:15:42,493 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/0c5061cd15a74e589919bce268988fd2, entries=150, sequenceid=542, filesize=30.5 K 2024-11-20T21:15:42,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/9382302d2c1b4381a0db8de3ea189d4f as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/9382302d2c1b4381a0db8de3ea189d4f 2024-11-20T21:15:42,499 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/9382302d2c1b4381a0db8de3ea189d4f, entries=150, sequenceid=542, filesize=12.0 K 2024-11-20T21:15:42,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/85f46fe9bc954be2810c857cc017ee81 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/85f46fe9bc954be2810c857cc017ee81 2024-11-20T21:15:42,503 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/85f46fe9bc954be2810c857cc017ee81, entries=150, sequenceid=542, filesize=12.0 K 2024-11-20T21:15:42,504 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 67de1f67a4e12320e34a07470b17ec6e in 1810ms, sequenceid=542, compaction requested=false 2024-11-20T21:15:42,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:42,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:42,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-20T21:15:42,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-20T21:15:42,508 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-20T21:15:42,508 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9650 sec 2024-11-20T21:15:42,510 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.9710 sec 2024-11-20T21:15:42,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-20T21:15:42,687 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:15:42,688 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:15:42,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-20T21:15:42,690 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:15:42,691 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:15:42,691 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:15:42,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-20T21:15:42,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-20T21:15:42,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-20T21:15:42,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:42,844 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 67de1f67a4e12320e34a07470b17ec6e 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-20T21:15:42,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=A 2024-11-20T21:15:42,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:42,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=B 2024-11-20T21:15:42,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:42,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=C 2024-11-20T21:15:42,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:42,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b43e2d56a5e642e68cd6df84d440be3b_67de1f67a4e12320e34a07470b17ec6e is 50, key is test_row_0/A:col10/1732137341061/Put/seqid=0 2024-11-20T21:15:42,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742272_1448 (size=12454) 2024-11-20T21:15:42,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:42,859 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b43e2d56a5e642e68cd6df84d440be3b_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b43e2d56a5e642e68cd6df84d440be3b_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:42,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/6ebaeb159b3d45bda0ca35ffc8604499, store: [table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:42,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/6ebaeb159b3d45bda0ca35ffc8604499 is 175, key is test_row_0/A:col10/1732137341061/Put/seqid=0 2024-11-20T21:15:42,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742273_1449 (size=31255) 2024-11-20T21:15:43,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-20T21:15:43,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:43,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:43,264 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=579, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/6ebaeb159b3d45bda0ca35ffc8604499 2024-11-20T21:15:43,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/b12c6b06ef4f4551ba4f7ad986601699 is 50, key is test_row_0/B:col10/1732137341061/Put/seqid=0 2024-11-20T21:15:43,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742274_1450 (size=12301) 2024-11-20T21:15:43,276 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=579 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/b12c6b06ef4f4551ba4f7ad986601699 2024-11-20T21:15:43,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:43,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137403262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:43,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:43,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:43,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:43,280 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:43,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137403267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:43,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:43,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137403268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:43,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:43,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137403268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:43,281 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 0b16ab3f13b7,36987,1732137236536 for 'test_row_1' in 67de1f67a4e12320e34a07470b17ec6e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T21:15:43,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:43,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:43,282 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:43,282 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 0b16ab3f13b7,36987,1732137236536 for 'test_row_1' in 67de1f67a4e12320e34a07470b17ec6e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T21:15:43,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/9103b5dc798e40b7926a0e9c72162a1a is 50, key is test_row_0/C:col10/1732137341061/Put/seqid=0 2024-11-20T21:15:43,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:43,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:43,283 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:43,283 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 0b16ab3f13b7,36987,1732137236536 for 'test_row_2' in 67de1f67a4e12320e34a07470b17ec6e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T21:15:43,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:43,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:43,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:43,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742275_1451 (size=12301) 2024-11-20T21:15:43,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:43,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137403297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:43,309 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 0b16ab3f13b7,36987,1732137236536 for 'test_row_2' in 67de1f67a4e12320e34a07470b17ec6e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T21:15:43,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:43,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:43,310 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:43,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-20T21:15:43,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:43,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137403387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:43,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:43,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:43,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:43,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:43,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137403607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:43,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:43,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:43,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:43,687 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=579 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/9103b5dc798e40b7926a0e9c72162a1a 2024-11-20T21:15:43,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/6ebaeb159b3d45bda0ca35ffc8604499 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/6ebaeb159b3d45bda0ca35ffc8604499 2024-11-20T21:15:43,695 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/6ebaeb159b3d45bda0ca35ffc8604499, entries=150, sequenceid=579, filesize=30.5 K 2024-11-20T21:15:43,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/b12c6b06ef4f4551ba4f7ad986601699 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/b12c6b06ef4f4551ba4f7ad986601699 2024-11-20T21:15:43,699 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/b12c6b06ef4f4551ba4f7ad986601699, entries=150, sequenceid=579, filesize=12.0 K 2024-11-20T21:15:43,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/9103b5dc798e40b7926a0e9c72162a1a as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/9103b5dc798e40b7926a0e9c72162a1a 2024-11-20T21:15:43,710 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/9103b5dc798e40b7926a0e9c72162a1a, entries=150, sequenceid=579, filesize=12.0 K 2024-11-20T21:15:43,713 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for 67de1f67a4e12320e34a07470b17ec6e in 869ms, sequenceid=579, compaction requested=true 2024-11-20T21:15:43,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:43,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:43,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-20T21:15:43,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-20T21:15:43,726 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-20T21:15:43,727 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0330 sec 2024-11-20T21:15:43,730 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.0410 sec 2024-11-20T21:15:43,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-20T21:15:43,828 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:15:43,828 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:15:43,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-20T21:15:43,830 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:15:43,832 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:15:43,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T21:15:43,832 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:15:43,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 67de1f67a4e12320e34a07470b17ec6e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T21:15:43,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=A 2024-11-20T21:15:43,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:43,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=B 2024-11-20T21:15:43,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:43,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:43,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=C 2024-11-20T21:15:43,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T21:15:43,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120efec5a7030ae43cda6bd5ec0cba95d3c_67de1f67a4e12320e34a07470b17ec6e is 50, key is test_row_0/A:col10/1732137343931/Put/seqid=0 2024-11-20T21:15:43,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742276_1452 (size=14994) 2024-11-20T21:15:43,958 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:43,961 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120efec5a7030ae43cda6bd5ec0cba95d3c_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120efec5a7030ae43cda6bd5ec0cba95d3c_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:43,962 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/a901ffad323846d1ab73b84b8697f2c7, store: [table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:43,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/a901ffad323846d1ab73b84b8697f2c7 is 175, key is test_row_0/A:col10/1732137343931/Put/seqid=0 2024-11-20T21:15:43,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T21:15:43,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:43,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:43,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:43,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:43,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:43,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:43,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742277_1453 (size=39949) 2024-11-20T21:15:43,998 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=591, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/a901ffad323846d1ab73b84b8697f2c7 2024-11-20T21:15:44,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/b7bcf29992334ef0ba00f9bef9af7e4a is 50, key is test_row_0/B:col10/1732137343931/Put/seqid=0 2024-11-20T21:15:44,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742278_1454 (size=12301) 2024-11-20T21:15:44,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T21:15:44,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T21:15:44,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:44,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:44,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:44,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:44,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:44,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:44,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:44,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137404234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:44,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:44,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:44,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:44,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T21:15:44,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:44,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:44,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:44,309 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:44,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:44,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:44,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:44,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137404357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:44,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:44,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:44,380 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:44,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T21:15:44,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T21:15:44,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:44,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:44,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:44,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:44,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:44,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:44,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/b7bcf29992334ef0ba00f9bef9af7e4a 2024-11-20T21:15:44,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/4b6816433078488fb6c1b40d3ac24a0f is 50, key is test_row_0/C:col10/1732137343931/Put/seqid=0 2024-11-20T21:15:44,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742279_1455 (size=12301) 2024-11-20T21:15:44,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:44,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137404587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:44,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/4b6816433078488fb6c1b40d3ac24a0f 2024-11-20T21:15:44,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:44,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:44,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:44,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T21:15:44,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:44,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:44,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:44,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:44,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:44,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:44,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/a901ffad323846d1ab73b84b8697f2c7 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/a901ffad323846d1ab73b84b8697f2c7 2024-11-20T21:15:44,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/a901ffad323846d1ab73b84b8697f2c7, entries=200, sequenceid=591, filesize=39.0 K 2024-11-20T21:15:44,648 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-20T21:15:44,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/b7bcf29992334ef0ba00f9bef9af7e4a as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/b7bcf29992334ef0ba00f9bef9af7e4a 2024-11-20T21:15:44,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/b7bcf29992334ef0ba00f9bef9af7e4a, entries=150, sequenceid=591, filesize=12.0 K 2024-11-20T21:15:44,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/4b6816433078488fb6c1b40d3ac24a0f as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/4b6816433078488fb6c1b40d3ac24a0f 2024-11-20T21:15:44,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/4b6816433078488fb6c1b40d3ac24a0f, entries=150, sequenceid=591, filesize=12.0 K 2024-11-20T21:15:44,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 67de1f67a4e12320e34a07470b17ec6e in 730ms, sequenceid=591, compaction requested=true 2024-11-20T21:15:44,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:44,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 67de1f67a4e12320e34a07470b17ec6e:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:15:44,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:44,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 67de1f67a4e12320e34a07470b17ec6e:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:15:44,664 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:15:44,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:44,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 67de1f67a4e12320e34a07470b17ec6e:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:15:44,664 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T21:15:44,664 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:15:44,665 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:15:44,665 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 67de1f67a4e12320e34a07470b17ec6e/A is initiating minor compaction (all files) 2024-11-20T21:15:44,665 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 67de1f67a4e12320e34a07470b17ec6e/A in TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:44,665 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/ccacf18537da4821947bfdf245017eed, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/0c5061cd15a74e589919bce268988fd2, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/6ebaeb159b3d45bda0ca35ffc8604499, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/a901ffad323846d1ab73b84b8697f2c7] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp, totalSize=131.0 K 2024-11-20T21:15:44,665 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:44,665 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. files: [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/ccacf18537da4821947bfdf245017eed, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/0c5061cd15a74e589919bce268988fd2, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/6ebaeb159b3d45bda0ca35ffc8604499, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/a901ffad323846d1ab73b84b8697f2c7] 2024-11-20T21:15:44,666 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:15:44,666 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccacf18537da4821947bfdf245017eed, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1732137339556 2024-11-20T21:15:44,666 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 67de1f67a4e12320e34a07470b17ec6e/B is initiating minor compaction (all files) 2024-11-20T21:15:44,666 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 67de1f67a4e12320e34a07470b17ec6e/B in TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:44,666 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/1db4b67f65d94d5fa871f2cf96dbd858, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/9382302d2c1b4381a0db8de3ea189d4f, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/b12c6b06ef4f4551ba4f7ad986601699, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/b7bcf29992334ef0ba00f9bef9af7e4a] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp, totalSize=48.5 K 2024-11-20T21:15:44,666 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c5061cd15a74e589919bce268988fd2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=542, earliestPutTs=1732137339705 2024-11-20T21:15:44,666 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 1db4b67f65d94d5fa871f2cf96dbd858, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1732137339556 2024-11-20T21:15:44,667 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ebaeb159b3d45bda0ca35ffc8604499, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=579, earliestPutTs=1732137341056 2024-11-20T21:15:44,667 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting a901ffad323846d1ab73b84b8697f2c7, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1732137343210 2024-11-20T21:15:44,667 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 9382302d2c1b4381a0db8de3ea189d4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=542, earliestPutTs=1732137339705 2024-11-20T21:15:44,667 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting b12c6b06ef4f4551ba4f7ad986601699, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=579, earliestPutTs=1732137341056 2024-11-20T21:15:44,668 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting b7bcf29992334ef0ba00f9bef9af7e4a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1732137343210 2024-11-20T21:15:44,681 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:44,689 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 67de1f67a4e12320e34a07470b17ec6e#B#compaction#384 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:44,690 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/2187fc64e2e04f17abebb9f75378ee3b is 50, key is test_row_0/B:col10/1732137343931/Put/seqid=0 2024-11-20T21:15:44,694 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120a8450af852634d06a94bbcc0637a8bfe_67de1f67a4e12320e34a07470b17ec6e store=[table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:44,697 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120a8450af852634d06a94bbcc0637a8bfe_67de1f67a4e12320e34a07470b17ec6e, store=[table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:44,697 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a8450af852634d06a94bbcc0637a8bfe_67de1f67a4e12320e34a07470b17ec6e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:44,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742280_1456 (size=4469) 2024-11-20T21:15:44,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742281_1457 (size=12881) 2024-11-20T21:15:44,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-20T21:15:44,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:44,773 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 67de1f67a4e12320e34a07470b17ec6e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T21:15:44,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=A 2024-11-20T21:15:44,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:44,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=B 2024-11-20T21:15:44,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:44,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=C 2024-11-20T21:15:44,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:44,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206960774cccf447fcb647dcbeddbb1966_67de1f67a4e12320e34a07470b17ec6e is 50, key is test_row_0/A:col10/1732137344228/Put/seqid=0 2024-11-20T21:15:44,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742282_1458 (size=12454) 2024-11-20T21:15:44,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:44,813 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206960774cccf447fcb647dcbeddbb1966_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206960774cccf447fcb647dcbeddbb1966_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:44,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/ede2af88356448f6aac661a6a63ad109, store: [table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:44,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/ede2af88356448f6aac661a6a63ad109 is 175, key is test_row_0/A:col10/1732137344228/Put/seqid=0 2024-11-20T21:15:44,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742283_1459 (size=31255) 2024-11-20T21:15:44,849 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=611, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/ede2af88356448f6aac661a6a63ad109 2024-11-20T21:15:44,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/72613ae97e57446582eaba1f4635ded9 is 50, key is test_row_0/B:col10/1732137344228/Put/seqid=0 2024-11-20T21:15:44,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:44,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:44,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742284_1460 (size=12301) 2024-11-20T21:15:44,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T21:15:45,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:45,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137405043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:45,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:45,063 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:45,128 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 67de1f67a4e12320e34a07470b17ec6e#A#compaction#383 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:45,129 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/9b6e3c03bf2143cfaf82b59b81ecf48a is 175, key is test_row_0/A:col10/1732137343931/Put/seqid=0 2024-11-20T21:15:45,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742285_1461 (size=31835) 2024-11-20T21:15:45,166 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/9b6e3c03bf2143cfaf82b59b81ecf48a as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/9b6e3c03bf2143cfaf82b59b81ecf48a 2024-11-20T21:15:45,169 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/2187fc64e2e04f17abebb9f75378ee3b as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/2187fc64e2e04f17abebb9f75378ee3b 2024-11-20T21:15:45,177 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 67de1f67a4e12320e34a07470b17ec6e/B of 67de1f67a4e12320e34a07470b17ec6e into 2187fc64e2e04f17abebb9f75378ee3b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:45,177 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:45,177 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., storeName=67de1f67a4e12320e34a07470b17ec6e/B, priority=12, startTime=1732137344664; duration=0sec 2024-11-20T21:15:45,177 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:45,177 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 67de1f67a4e12320e34a07470b17ec6e:B 2024-11-20T21:15:45,178 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 67de1f67a4e12320e34a07470b17ec6e/A of 67de1f67a4e12320e34a07470b17ec6e into 9b6e3c03bf2143cfaf82b59b81ecf48a(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:45,178 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:45,178 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., storeName=67de1f67a4e12320e34a07470b17ec6e/A, priority=12, startTime=1732137344663; duration=0sec 2024-11-20T21:15:45,178 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:45,178 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 67de1f67a4e12320e34a07470b17ec6e:A 2024-11-20T21:15:45,178 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:15:45,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:45,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137405168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:45,178 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:45,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:45,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:45,182 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49648 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:15:45,182 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 67de1f67a4e12320e34a07470b17ec6e/C is initiating minor compaction (all files) 2024-11-20T21:15:45,182 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 67de1f67a4e12320e34a07470b17ec6e/C in TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:45,182 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/2b4382dd717c4485adbc57cf3f3ad2d2, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/85f46fe9bc954be2810c857cc017ee81, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/9103b5dc798e40b7926a0e9c72162a1a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/4b6816433078488fb6c1b40d3ac24a0f] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp, totalSize=48.5 K 2024-11-20T21:15:45,182 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b4382dd717c4485adbc57cf3f3ad2d2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=531, earliestPutTs=1732137339556 2024-11-20T21:15:45,183 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85f46fe9bc954be2810c857cc017ee81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=542, earliestPutTs=1732137339705 2024-11-20T21:15:45,183 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9103b5dc798e40b7926a0e9c72162a1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=579, earliestPutTs=1732137341056 2024-11-20T21:15:45,183 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b6816433078488fb6c1b40d3ac24a0f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1732137343210 2024-11-20T21:15:45,198 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 67de1f67a4e12320e34a07470b17ec6e#C#compaction#387 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:45,198 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/cd5fbeb805f6439db2872cd902785add is 50, key is test_row_0/C:col10/1732137343931/Put/seqid=0 2024-11-20T21:15:45,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742286_1462 (size=12881) 2024-11-20T21:15:45,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:45,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137405297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:45,311 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 0b16ab3f13b7,36987,1732137236536 for 'test_row_1' in 67de1f67a4e12320e34a07470b17ec6e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T21:15:45,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:45,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:45,312 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:45,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:45,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137405297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:45,313 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 0b16ab3f13b7,36987,1732137236536 for 'test_row_1' in 67de1f67a4e12320e34a07470b17ec6e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T21:15:45,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:45,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:45,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:45,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137405298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:45,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:45,314 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 0b16ab3f13b7,36987,1732137236536 for 'test_row_2' in 67de1f67a4e12320e34a07470b17ec6e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T21:15:45,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:45,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:45,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:45,315 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=611 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/72613ae97e57446582eaba1f4635ded9 2024-11-20T21:15:45,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/480da8157eae439dbdcf7c17318b4571 is 50, key is test_row_0/C:col10/1732137344228/Put/seqid=0 2024-11-20T21:15:45,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:45,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137405317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:45,333 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 0b16ab3f13b7,36987,1732137236536 for 'test_row_2' in 67de1f67a4e12320e34a07470b17ec6e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4271 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T21:15:45,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:45,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:45,333 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:45,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742287_1463 (size=12301) 2024-11-20T21:15:45,379 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=611 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/480da8157eae439dbdcf7c17318b4571 2024-11-20T21:15:45,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/ede2af88356448f6aac661a6a63ad109 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/ede2af88356448f6aac661a6a63ad109 2024-11-20T21:15:45,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:45,411 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/ede2af88356448f6aac661a6a63ad109, entries=150, sequenceid=611, filesize=30.5 K 2024-11-20T21:15:45,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137405387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:45,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/72613ae97e57446582eaba1f4635ded9 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/72613ae97e57446582eaba1f4635ded9 2024-11-20T21:15:45,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:45,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:45,412 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:45,420 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/72613ae97e57446582eaba1f4635ded9, entries=150, sequenceid=611, filesize=12.0 K 2024-11-20T21:15:45,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/480da8157eae439dbdcf7c17318b4571 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/480da8157eae439dbdcf7c17318b4571 2024-11-20T21:15:45,431 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/480da8157eae439dbdcf7c17318b4571, entries=150, sequenceid=611, filesize=12.0 K 2024-11-20T21:15:45,433 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 67de1f67a4e12320e34a07470b17ec6e in 660ms, sequenceid=611, compaction requested=false 2024-11-20T21:15:45,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:45,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:45,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-20T21:15:45,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-20T21:15:45,443 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-20T21:15:45,443 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6040 sec 2024-11-20T21:15:45,447 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.6160 sec 2024-11-20T21:15:45,669 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/cd5fbeb805f6439db2872cd902785add as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/cd5fbeb805f6439db2872cd902785add 2024-11-20T21:15:45,678 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 67de1f67a4e12320e34a07470b17ec6e/C of 67de1f67a4e12320e34a07470b17ec6e into cd5fbeb805f6439db2872cd902785add(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:45,678 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:45,678 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., storeName=67de1f67a4e12320e34a07470b17ec6e/C, priority=12, startTime=1732137344664; duration=0sec 2024-11-20T21:15:45,678 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:45,678 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 67de1f67a4e12320e34a07470b17ec6e:C 2024-11-20T21:15:45,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:45,722 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 67de1f67a4e12320e34a07470b17ec6e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-20T21:15:45,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=A 2024-11-20T21:15:45,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:45,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=B 2024-11-20T21:15:45,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:45,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=C 2024-11-20T21:15:45,723 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:45,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208345a22ffd5645f4b945152fdada76dc_67de1f67a4e12320e34a07470b17ec6e is 50, key is test_row_0/A:col10/1732137345721/Put/seqid=0 2024-11-20T21:15:45,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742288_1464 (size=14994) 2024-11-20T21:15:45,750 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:45,753 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411208345a22ffd5645f4b945152fdada76dc_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208345a22ffd5645f4b945152fdada76dc_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:45,754 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/819fde83a74e4f36908393f193dd3a15, store: [table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:45,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/819fde83a74e4f36908393f193dd3a15 is 175, key is test_row_0/A:col10/1732137345721/Put/seqid=0 2024-11-20T21:15:45,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742289_1465 (size=39949) 2024-11-20T21:15:45,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:45,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137405872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:45,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:45,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:45,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:45,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-20T21:15:45,977 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:15:45,977 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:15:45,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-20T21:15:45,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T21:15:45,979 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:15:45,980 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:15:45,980 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:15:46,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:46,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137405997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:46,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:46,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:46,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:46,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T21:15:46,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T21:15:46,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:46,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:46,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:46,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:46,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:46,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:46,158 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=627, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/819fde83a74e4f36908393f193dd3a15 2024-11-20T21:15:46,171 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/9d95a19e4dc04339868cc7a41dbc86cc is 50, key is test_row_0/B:col10/1732137345721/Put/seqid=0 2024-11-20T21:15:46,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742290_1466 (size=12301) 2024-11-20T21:15:46,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:46,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137406207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:46,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:46,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:46,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:46,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T21:15:46,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:46,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:46,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:46,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:46,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:46,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:46,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T21:15:46,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T21:15:46,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:46,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:46,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:46,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:46,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:46,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:46,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:46,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137406536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:46,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:46,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:46,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:46,586 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=627 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/9d95a19e4dc04339868cc7a41dbc86cc 2024-11-20T21:15:46,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T21:15:46,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:46,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:46,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:46,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:46,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:46,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:46,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/a789f254701a4954a590751d26b9b97c is 50, key is test_row_0/C:col10/1732137345721/Put/seqid=0 2024-11-20T21:15:46,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T21:15:46,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742291_1467 (size=12301) 2024-11-20T21:15:46,621 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=627 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/a789f254701a4954a590751d26b9b97c 2024-11-20T21:15:46,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/819fde83a74e4f36908393f193dd3a15 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/819fde83a74e4f36908393f193dd3a15 2024-11-20T21:15:46,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/819fde83a74e4f36908393f193dd3a15, entries=200, sequenceid=627, filesize=39.0 K 2024-11-20T21:15:46,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/9d95a19e4dc04339868cc7a41dbc86cc as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/9d95a19e4dc04339868cc7a41dbc86cc 2024-11-20T21:15:46,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/9d95a19e4dc04339868cc7a41dbc86cc, entries=150, sequenceid=627, filesize=12.0 K 2024-11-20T21:15:46,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/a789f254701a4954a590751d26b9b97c as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/a789f254701a4954a590751d26b9b97c 2024-11-20T21:15:46,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/a789f254701a4954a590751d26b9b97c, entries=150, sequenceid=627, filesize=12.0 K 2024-11-20T21:15:46,655 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 67de1f67a4e12320e34a07470b17ec6e in 933ms, sequenceid=627, compaction requested=true 2024-11-20T21:15:46,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:46,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 67de1f67a4e12320e34a07470b17ec6e:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:15:46,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:46,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 67de1f67a4e12320e34a07470b17ec6e:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:15:46,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:46,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 67de1f67a4e12320e34a07470b17ec6e:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:15:46,655 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:46,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T21:15:46,657 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:46,661 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:46,661 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 67de1f67a4e12320e34a07470b17ec6e/A is initiating minor compaction (all files) 2024-11-20T21:15:46,661 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 67de1f67a4e12320e34a07470b17ec6e/A in TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:46,661 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/9b6e3c03bf2143cfaf82b59b81ecf48a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/ede2af88356448f6aac661a6a63ad109, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/819fde83a74e4f36908393f193dd3a15] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp, totalSize=100.6 K 2024-11-20T21:15:46,661 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:46,661 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. files: [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/9b6e3c03bf2143cfaf82b59b81ecf48a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/ede2af88356448f6aac661a6a63ad109, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/819fde83a74e4f36908393f193dd3a15] 2024-11-20T21:15:46,661 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b6e3c03bf2143cfaf82b59b81ecf48a, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1732137343210 2024-11-20T21:15:46,662 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting ede2af88356448f6aac661a6a63ad109, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=611, earliestPutTs=1732137344121 2024-11-20T21:15:46,662 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 819fde83a74e4f36908393f193dd3a15, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=627, earliestPutTs=1732137345002 2024-11-20T21:15:46,670 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:46,672 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:46,672 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 67de1f67a4e12320e34a07470b17ec6e/B is initiating minor compaction (all files) 2024-11-20T21:15:46,673 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 67de1f67a4e12320e34a07470b17ec6e/B in TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:46,673 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/2187fc64e2e04f17abebb9f75378ee3b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/72613ae97e57446582eaba1f4635ded9, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/9d95a19e4dc04339868cc7a41dbc86cc] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp, totalSize=36.6 K 2024-11-20T21:15:46,680 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 2187fc64e2e04f17abebb9f75378ee3b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1732137343210 2024-11-20T21:15:46,681 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 72613ae97e57446582eaba1f4635ded9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=611, earliestPutTs=1732137344121 2024-11-20T21:15:46,681 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d95a19e4dc04339868cc7a41dbc86cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=627, earliestPutTs=1732137345002 2024-11-20T21:15:46,693 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120b27f3d76f38e4b92892dbb61cb34fda9_67de1f67a4e12320e34a07470b17ec6e store=[table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:46,695 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120b27f3d76f38e4b92892dbb61cb34fda9_67de1f67a4e12320e34a07470b17ec6e, store=[table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:46,695 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b27f3d76f38e4b92892dbb61cb34fda9_67de1f67a4e12320e34a07470b17ec6e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:46,696 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 67de1f67a4e12320e34a07470b17ec6e#B#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:46,696 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/ea225361d3334ee8930e58e471889c8b is 50, key is test_row_0/B:col10/1732137345721/Put/seqid=0 2024-11-20T21:15:46,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742293_1469 (size=4469) 2024-11-20T21:15:46,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742292_1468 (size=12983) 2024-11-20T21:15:46,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-20T21:15:46,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:46,755 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 67de1f67a4e12320e34a07470b17ec6e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-20T21:15:46,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=A 2024-11-20T21:15:46,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:46,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=B 2024-11-20T21:15:46,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:46,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=C 2024-11-20T21:15:46,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:46,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120151ab4f2c7af4f42908ddc8a968a3b21_67de1f67a4e12320e34a07470b17ec6e is 50, key is test_row_0/A:col10/1732137345840/Put/seqid=0 2024-11-20T21:15:46,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742294_1470 (size=12454) 2024-11-20T21:15:46,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:46,770 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120151ab4f2c7af4f42908ddc8a968a3b21_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120151ab4f2c7af4f42908ddc8a968a3b21_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:46,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/a480ba0999c44c928fc6692f162c3236, store: [table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:46,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/a480ba0999c44c928fc6692f162c3236 is 175, key is test_row_0/A:col10/1732137345840/Put/seqid=0 2024-11-20T21:15:46,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742295_1471 (size=31255) 2024-11-20T21:15:47,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:47,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:47,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T21:15:47,140 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 67de1f67a4e12320e34a07470b17ec6e#A#compaction#392 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:47,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:47,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137407126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:47,141 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/6c33431948b049bfa57aae52f6522522 is 175, key is test_row_0/A:col10/1732137345721/Put/seqid=0 2024-11-20T21:15:47,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:47,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:47,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:47,148 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/ea225361d3334ee8930e58e471889c8b as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/ea225361d3334ee8930e58e471889c8b 2024-11-20T21:15:47,151 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 67de1f67a4e12320e34a07470b17ec6e/B of 67de1f67a4e12320e34a07470b17ec6e into ea225361d3334ee8930e58e471889c8b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:47,151 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:47,151 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., storeName=67de1f67a4e12320e34a07470b17ec6e/B, priority=13, startTime=1732137346655; duration=0sec 2024-11-20T21:15:47,151 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:47,151 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 67de1f67a4e12320e34a07470b17ec6e:B 2024-11-20T21:15:47,151 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:47,152 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:47,152 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 67de1f67a4e12320e34a07470b17ec6e/C is initiating minor compaction (all files) 2024-11-20T21:15:47,152 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 67de1f67a4e12320e34a07470b17ec6e/C in TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:47,152 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/cd5fbeb805f6439db2872cd902785add, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/480da8157eae439dbdcf7c17318b4571, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/a789f254701a4954a590751d26b9b97c] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp, totalSize=36.6 K 2024-11-20T21:15:47,153 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting cd5fbeb805f6439db2872cd902785add, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1732137343210 2024-11-20T21:15:47,156 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 480da8157eae439dbdcf7c17318b4571, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=611, earliestPutTs=1732137344121 2024-11-20T21:15:47,156 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting a789f254701a4954a590751d26b9b97c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=627, earliestPutTs=1732137345002 2024-11-20T21:15:47,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742296_1472 (size=31937) 2024-11-20T21:15:47,162 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/6c33431948b049bfa57aae52f6522522 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/6c33431948b049bfa57aae52f6522522 2024-11-20T21:15:47,170 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 67de1f67a4e12320e34a07470b17ec6e/A of 67de1f67a4e12320e34a07470b17ec6e into 6c33431948b049bfa57aae52f6522522(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:47,170 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:47,170 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., storeName=67de1f67a4e12320e34a07470b17ec6e/A, priority=13, startTime=1732137346655; duration=0sec 2024-11-20T21:15:47,170 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:47,170 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 67de1f67a4e12320e34a07470b17ec6e:A 2024-11-20T21:15:47,171 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 67de1f67a4e12320e34a07470b17ec6e#C#compaction#395 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:47,171 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/58739e6837934ba699bac9864577788b is 50, key is test_row_0/C:col10/1732137345721/Put/seqid=0 2024-11-20T21:15:47,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742297_1473 (size=12983) 2024-11-20T21:15:47,178 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=646, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/a480ba0999c44c928fc6692f162c3236 2024-11-20T21:15:47,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/645d782495b6400ab2899251071e702e is 50, key is test_row_0/B:col10/1732137345840/Put/seqid=0 2024-11-20T21:15:47,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742298_1474 (size=12301) 2024-11-20T21:15:47,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:47,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137407248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:47,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:47,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:47,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:47,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:47,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137407476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:47,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:47,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:47,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:47,581 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/58739e6837934ba699bac9864577788b as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/58739e6837934ba699bac9864577788b 2024-11-20T21:15:47,616 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=646 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/645d782495b6400ab2899251071e702e 2024-11-20T21:15:47,617 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 67de1f67a4e12320e34a07470b17ec6e/C of 67de1f67a4e12320e34a07470b17ec6e into 58739e6837934ba699bac9864577788b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:47,617 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:47,617 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., storeName=67de1f67a4e12320e34a07470b17ec6e/C, priority=13, startTime=1732137346655; duration=0sec 2024-11-20T21:15:47,617 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:47,617 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 67de1f67a4e12320e34a07470b17ec6e:C 2024-11-20T21:15:47,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/1136c3c275aa486296376b75dd7a3fc8 is 50, key is test_row_0/C:col10/1732137345840/Put/seqid=0 2024-11-20T21:15:47,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742299_1475 (size=12301) 2024-11-20T21:15:47,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:47,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137407797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:47,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:47,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:47,807 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:48,089 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=646 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/1136c3c275aa486296376b75dd7a3fc8 2024-11-20T21:15:48,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/a480ba0999c44c928fc6692f162c3236 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/a480ba0999c44c928fc6692f162c3236 2024-11-20T21:15:48,098 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/a480ba0999c44c928fc6692f162c3236, entries=150, sequenceid=646, filesize=30.5 K 2024-11-20T21:15:48,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/645d782495b6400ab2899251071e702e as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/645d782495b6400ab2899251071e702e 2024-11-20T21:15:48,102 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/645d782495b6400ab2899251071e702e, entries=150, sequenceid=646, filesize=12.0 K 2024-11-20T21:15:48,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/1136c3c275aa486296376b75dd7a3fc8 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/1136c3c275aa486296376b75dd7a3fc8 2024-11-20T21:15:48,106 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/1136c3c275aa486296376b75dd7a3fc8, entries=150, sequenceid=646, filesize=12.0 K 2024-11-20T21:15:48,106 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 67de1f67a4e12320e34a07470b17ec6e in 1351ms, sequenceid=646, compaction requested=false 2024-11-20T21:15:48,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:48,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:48,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-20T21:15:48,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-20T21:15:48,110 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-20T21:15:48,110 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1280 sec 2024-11-20T21:15:48,112 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.1330 sec 2024-11-20T21:15:48,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-20T21:15:48,127 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:15:48,127 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:15:48,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-20T21:15:48,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T21:15:48,128 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:15:48,129 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:15:48,129 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:15:48,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T21:15:48,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-20T21:15:48,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:48,281 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 67de1f67a4e12320e34a07470b17ec6e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-20T21:15:48,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=A 2024-11-20T21:15:48,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:48,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=B 2024-11-20T21:15:48,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:48,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=C 2024-11-20T21:15:48,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:48,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120740d9d15530e4e88993fb921b26e0d90_67de1f67a4e12320e34a07470b17ec6e is 50, key is test_row_0/A:col10/1732137347107/Put/seqid=0 2024-11-20T21:15:48,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742300_1476 (size=12454) 2024-11-20T21:15:48,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:48,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:48,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T21:15:48,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:48,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137408481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:48,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:48,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:48,488 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:48,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:48,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137408598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:48,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:48,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:48,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:48,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:48,708 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120740d9d15530e4e88993fb921b26e0d90_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120740d9d15530e4e88993fb921b26e0d90_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:48,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/3250b45f058a41abba8671ec4cb4cae0, store: [table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:48,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/3250b45f058a41abba8671ec4cb4cae0 is 175, key is test_row_0/A:col10/1732137347107/Put/seqid=0 2024-11-20T21:15:48,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742301_1477 (size=31255) 2024-11-20T21:15:48,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T21:15:48,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:48,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137408817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:48,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:48,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:48,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:49,121 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=662, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/3250b45f058a41abba8671ec4cb4cae0 2024-11-20T21:15:49,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:49,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137409127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:49,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:49,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:49,139 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:49,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/18ad840e329347e390ee250747e62b1a is 50, key is test_row_0/B:col10/1732137347107/Put/seqid=0 2024-11-20T21:15:49,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742302_1478 (size=12301) 2024-11-20T21:15:49,172 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=662 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/18ad840e329347e390ee250747e62b1a 2024-11-20T21:15:49,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/38b65ac0e3564ef88bbc9153c213173f is 50, key is test_row_0/C:col10/1732137347107/Put/seqid=0 2024-11-20T21:15:49,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742303_1479 (size=12301) 2024-11-20T21:15:49,201 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=662 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/38b65ac0e3564ef88bbc9153c213173f 2024-11-20T21:15:49,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/3250b45f058a41abba8671ec4cb4cae0 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/3250b45f058a41abba8671ec4cb4cae0 2024-11-20T21:15:49,224 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/3250b45f058a41abba8671ec4cb4cae0, entries=150, sequenceid=662, filesize=30.5 K 2024-11-20T21:15:49,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/18ad840e329347e390ee250747e62b1a as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/18ad840e329347e390ee250747e62b1a 2024-11-20T21:15:49,245 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/18ad840e329347e390ee250747e62b1a, entries=150, sequenceid=662, filesize=12.0 K 2024-11-20T21:15:49,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/38b65ac0e3564ef88bbc9153c213173f as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/38b65ac0e3564ef88bbc9153c213173f 2024-11-20T21:15:49,252 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/38b65ac0e3564ef88bbc9153c213173f, entries=150, sequenceid=662, filesize=12.0 K 2024-11-20T21:15:49,256 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 67de1f67a4e12320e34a07470b17ec6e in 975ms, sequenceid=662, compaction requested=true 2024-11-20T21:15:49,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:49,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:49,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-20T21:15:49,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-20T21:15:49,259 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-20T21:15:49,259 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1280 sec 2024-11-20T21:15:49,261 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.1330 sec 2024-11-20T21:15:49,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-20T21:15:49,267 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:15:49,268 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:15:49,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-20T21:15:49,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T21:15:49,270 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:15:49,271 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:15:49,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:15:49,366 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 67de1f67a4e12320e34a07470b17ec6e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-20T21:15:49,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=A 2024-11-20T21:15:49,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:49,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=B 2024-11-20T21:15:49,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:49,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=C 2024-11-20T21:15:49,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:49,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:49,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T21:15:49,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202aa814b8744542c39f9b048054fa2079_67de1f67a4e12320e34a07470b17ec6e is 50, key is test_row_0/A:col10/1732137348479/Put/seqid=0 2024-11-20T21:15:49,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742304_1480 (size=14994) 2024-11-20T21:15:49,387 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:49,395 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411202aa814b8744542c39f9b048054fa2079_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202aa814b8744542c39f9b048054fa2079_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:49,396 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/06897bfc52854e33959306a0a05387dc, store: [table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:49,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/06897bfc52854e33959306a0a05387dc is 175, key is test_row_0/A:col10/1732137348479/Put/seqid=0 2024-11-20T21:15:49,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742305_1481 (size=39949) 2024-11-20T21:15:49,406 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=682, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/06897bfc52854e33959306a0a05387dc 2024-11-20T21:15:49,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:49,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137409402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:49,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/4e966c9a107240d295333c583b9c6fe9 is 50, key is test_row_0/B:col10/1732137348479/Put/seqid=0 2024-11-20T21:15:49,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:49,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:49,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:49,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742306_1482 (size=12301) 2024-11-20T21:15:49,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:49,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137409406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:49,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-20T21:15:49,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:49,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137409409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:49,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:49,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:49,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:49,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:49,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:49,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:49,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:49,423 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:49,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:49,423 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:49,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:49,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:49,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137409413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:49,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:49,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:49,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:49,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:49,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:49,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137409516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:49,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:49,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:49,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:49,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:49,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137409527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:49,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:49,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:49,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137409527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:49,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:49,540 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:49,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:49,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:49,540 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:49,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137409536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:49,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:49,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:49,548 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:49,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-20T21:15:49,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:49,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:49,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:49,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:49,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:49,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:49,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T21:15:49,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:49,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38288 deadline: 1732137409647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:49,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:49,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:49,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:49,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-20T21:15:49,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:49,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:49,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:49,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:49,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:49,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:49,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38358 deadline: 1732137409736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:49,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:49,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:49,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:49,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:49,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38342 deadline: 1732137409747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:49,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:49,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:49,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:49,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:49,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38336 deadline: 1732137409749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:49,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:49,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:49,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:49,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38284 deadline: 1732137409759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:49,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:49,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:49,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:15:49,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=682 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/4e966c9a107240d295333c583b9c6fe9 2024-11-20T21:15:49,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/e921ec23fad6408fa7a8ea02852b0474 is 50, key is test_row_0/C:col10/1732137348479/Put/seqid=0 2024-11-20T21:15:49,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742307_1483 (size=12301) 2024-11-20T21:15:49,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=682 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/e921ec23fad6408fa7a8ea02852b0474 2024-11-20T21:15:49,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/06897bfc52854e33959306a0a05387dc as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/06897bfc52854e33959306a0a05387dc 2024-11-20T21:15:49,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/06897bfc52854e33959306a0a05387dc, entries=200, sequenceid=682, filesize=39.0 K 2024-11-20T21:15:49,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/4e966c9a107240d295333c583b9c6fe9 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/4e966c9a107240d295333c583b9c6fe9 2024-11-20T21:15:49,863 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/4e966c9a107240d295333c583b9c6fe9, entries=150, sequenceid=682, filesize=12.0 K 2024-11-20T21:15:49,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/e921ec23fad6408fa7a8ea02852b0474 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/e921ec23fad6408fa7a8ea02852b0474 2024-11-20T21:15:49,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/e921ec23fad6408fa7a8ea02852b0474, entries=150, sequenceid=682, filesize=12.0 K 2024-11-20T21:15:49,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 67de1f67a4e12320e34a07470b17ec6e in 504ms, sequenceid=682, compaction requested=true 2024-11-20T21:15:49,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:49,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 67de1f67a4e12320e34a07470b17ec6e:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:15:49,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:49,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 67de1f67a4e12320e34a07470b17ec6e:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:15:49,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:49,871 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:15:49,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 67de1f67a4e12320e34a07470b17ec6e:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:15:49,871 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:49,871 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:15:49,872 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134396 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:15:49,872 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:15:49,872 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 67de1f67a4e12320e34a07470b17ec6e/A is initiating minor compaction (all files) 2024-11-20T21:15:49,872 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 67de1f67a4e12320e34a07470b17ec6e/B is initiating minor compaction (all files) 2024-11-20T21:15:49,872 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 67de1f67a4e12320e34a07470b17ec6e/B in TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:49,872 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 67de1f67a4e12320e34a07470b17ec6e/A in TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:49,872 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/ea225361d3334ee8930e58e471889c8b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/645d782495b6400ab2899251071e702e, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/18ad840e329347e390ee250747e62b1a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/4e966c9a107240d295333c583b9c6fe9] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp, totalSize=48.7 K 2024-11-20T21:15:49,872 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/6c33431948b049bfa57aae52f6522522, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/a480ba0999c44c928fc6692f162c3236, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/3250b45f058a41abba8671ec4cb4cae0, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/06897bfc52854e33959306a0a05387dc] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp, totalSize=131.2 K 2024-11-20T21:15:49,872 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:49,872 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. files: [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/6c33431948b049bfa57aae52f6522522, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/a480ba0999c44c928fc6692f162c3236, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/3250b45f058a41abba8671ec4cb4cae0, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/06897bfc52854e33959306a0a05387dc] 2024-11-20T21:15:49,873 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting ea225361d3334ee8930e58e471889c8b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=627, earliestPutTs=1732137345002 2024-11-20T21:15:49,874 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c33431948b049bfa57aae52f6522522, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=627, earliestPutTs=1732137345002 2024-11-20T21:15:49,874 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 645d782495b6400ab2899251071e702e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=646, earliestPutTs=1732137345823 2024-11-20T21:15:49,874 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting a480ba0999c44c928fc6692f162c3236, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=646, earliestPutTs=1732137345823 2024-11-20T21:15:49,874 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 18ad840e329347e390ee250747e62b1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=662, earliestPutTs=1732137347091 2024-11-20T21:15:49,875 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3250b45f058a41abba8671ec4cb4cae0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=662, earliestPutTs=1732137347091 2024-11-20T21:15:49,875 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e966c9a107240d295333c583b9c6fe9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=682, earliestPutTs=1732137348453 2024-11-20T21:15:49,875 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06897bfc52854e33959306a0a05387dc, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=682, earliestPutTs=1732137348449 2024-11-20T21:15:49,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T21:15:49,897 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:49,900 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411209272167dd67441a7b9a5c0b358faecba_67de1f67a4e12320e34a07470b17ec6e store=[table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:49,903 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411209272167dd67441a7b9a5c0b358faecba_67de1f67a4e12320e34a07470b17ec6e, store=[table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:49,903 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209272167dd67441a7b9a5c0b358faecba_67de1f67a4e12320e34a07470b17ec6e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:49,903 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 67de1f67a4e12320e34a07470b17ec6e#B#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:49,904 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/d33bcaca66d7409bb950178842fa9cd8 is 50, key is test_row_0/B:col10/1732137348479/Put/seqid=0 2024-11-20T21:15:49,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-20T21:15:49,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:49,908 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 67de1f67a4e12320e34a07470b17ec6e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T21:15:49,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=A 2024-11-20T21:15:49,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:49,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=B 2024-11-20T21:15:49,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:49,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=C 2024-11-20T21:15:49,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:49,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742308_1484 (size=4469) 2024-11-20T21:15:49,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742309_1485 (size=13119) 2024-11-20T21:15:49,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f97586f3529549beac2f0d3e0c0d2a52_67de1f67a4e12320e34a07470b17ec6e is 50, key is test_row_0/A:col10/1732137349405/Put/seqid=0 2024-11-20T21:15:49,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742310_1486 (size=12454) 2024-11-20T21:15:49,949 INFO [Thread-1746 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1746. 2024-11-20T21:15:49,949 DEBUG [Thread-1746 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:15:49,949 DEBUG [Thread-1746 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:49,950 DEBUG [Thread-1746 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:49,950 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:15:49,953 INFO [Thread-1752 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1752. 2024-11-20T21:15:49,953 DEBUG [Thread-1752 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:15:49,953 DEBUG [Thread-1752 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:49,953 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:15:49,953 DEBUG [Thread-1752 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:49,955 INFO [Thread-1750 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1750. 2024-11-20T21:15:49,955 DEBUG [Thread-1750 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:15:49,955 DEBUG [Thread-1750 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:49,955 DEBUG [Thread-1750 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:49,955 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:15:49,961 INFO [Thread-1754 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1754. 2024-11-20T21:15:49,961 DEBUG [Thread-1754 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:15:49,961 DEBUG [Thread-1754 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:49,961 DEBUG [Thread-1754 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:49,961 INFO [Thread-1748 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1748. 2024-11-20T21:15:49,961 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:15:49,961 DEBUG [Thread-1748 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:15:49,961 DEBUG [Thread-1748 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:49,962 DEBUG [Thread-1748 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:49,962 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:15:50,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:50,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. as already flushing 2024-11-20T21:15:50,059 INFO [Thread-1743 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1743. 2024-11-20T21:15:50,059 DEBUG [Thread-1743 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:15:50,059 DEBUG [Thread-1743 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:50,059 DEBUG [Thread-1743 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:50,059 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:15:50,069 INFO [Thread-1739 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1739. 2024-11-20T21:15:50,069 DEBUG [Thread-1739 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:15:50,069 DEBUG [Thread-1739 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:50,069 DEBUG [Thread-1739 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:50,069 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:15:50,069 INFO [Thread-1735 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1735. 2024-11-20T21:15:50,069 DEBUG [Thread-1735 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:15:50,069 DEBUG [Thread-1735 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:50,069 DEBUG [Thread-1735 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:50,069 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:15:50,078 INFO [Thread-1737 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1737. 2024-11-20T21:15:50,078 DEBUG [Thread-1737 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:15:50,078 DEBUG [Thread-1737 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:50,078 DEBUG [Thread-1737 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:50,078 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:15:50,320 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 67de1f67a4e12320e34a07470b17ec6e#A#compaction#404 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:50,321 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/652850ec50dc4c9fad5d295f8c66da96 is 175, key is test_row_0/A:col10/1732137348479/Put/seqid=0 2024-11-20T21:15:50,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742311_1487 (size=32073) 2024-11-20T21:15:50,325 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/d33bcaca66d7409bb950178842fa9cd8 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/d33bcaca66d7409bb950178842fa9cd8 2024-11-20T21:15:50,329 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 67de1f67a4e12320e34a07470b17ec6e/B of 67de1f67a4e12320e34a07470b17ec6e into d33bcaca66d7409bb950178842fa9cd8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:50,329 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:50,329 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., storeName=67de1f67a4e12320e34a07470b17ec6e/B, priority=12, startTime=1732137349871; duration=0sec 2024-11-20T21:15:50,329 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:50,329 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 67de1f67a4e12320e34a07470b17ec6e:B 2024-11-20T21:15:50,329 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:15:50,330 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:15:50,330 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 67de1f67a4e12320e34a07470b17ec6e/C is initiating minor compaction (all files) 2024-11-20T21:15:50,330 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 67de1f67a4e12320e34a07470b17ec6e/C in TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:50,330 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/58739e6837934ba699bac9864577788b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/1136c3c275aa486296376b75dd7a3fc8, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/38b65ac0e3564ef88bbc9153c213173f, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/e921ec23fad6408fa7a8ea02852b0474] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp, totalSize=48.7 K 2024-11-20T21:15:50,330 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 58739e6837934ba699bac9864577788b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=627, earliestPutTs=1732137345002 2024-11-20T21:15:50,330 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 1136c3c275aa486296376b75dd7a3fc8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=646, earliestPutTs=1732137345823 2024-11-20T21:15:50,331 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 38b65ac0e3564ef88bbc9153c213173f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=662, earliestPutTs=1732137347091 2024-11-20T21:15:50,331 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting e921ec23fad6408fa7a8ea02852b0474, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=682, earliestPutTs=1732137348453 2024-11-20T21:15:50,338 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 67de1f67a4e12320e34a07470b17ec6e#C#compaction#407 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:50,338 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/806d5730038e4849b5e97a6d55e8324c is 50, key is test_row_0/C:col10/1732137348479/Put/seqid=0 2024-11-20T21:15:50,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742312_1488 (size=13119) 2024-11-20T21:15:50,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:50,346 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120f97586f3529549beac2f0d3e0c0d2a52_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f97586f3529549beac2f0d3e0c0d2a52_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:50,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/51048b5d1df74c6882611c3851fc9f71, store: [table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:50,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/51048b5d1df74c6882611c3851fc9f71 is 175, key is test_row_0/A:col10/1732137349405/Put/seqid=0 2024-11-20T21:15:50,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742313_1489 (size=31255) 2024-11-20T21:15:50,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T21:15:50,668 INFO [Thread-1741 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1741. 2024-11-20T21:15:50,668 DEBUG [Thread-1741 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:15:50,668 DEBUG [Thread-1741 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:50,668 DEBUG [Thread-1741 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:50,668 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:15:50,729 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/652850ec50dc4c9fad5d295f8c66da96 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/652850ec50dc4c9fad5d295f8c66da96 2024-11-20T21:15:50,733 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 67de1f67a4e12320e34a07470b17ec6e/A of 67de1f67a4e12320e34a07470b17ec6e into 652850ec50dc4c9fad5d295f8c66da96(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:50,733 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:50,733 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., storeName=67de1f67a4e12320e34a07470b17ec6e/A, priority=12, startTime=1732137349871; duration=0sec 2024-11-20T21:15:50,733 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:50,733 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 67de1f67a4e12320e34a07470b17ec6e:A 2024-11-20T21:15:50,744 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/806d5730038e4849b5e97a6d55e8324c as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/806d5730038e4849b5e97a6d55e8324c 2024-11-20T21:15:50,747 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 67de1f67a4e12320e34a07470b17ec6e/C of 67de1f67a4e12320e34a07470b17ec6e into 806d5730038e4849b5e97a6d55e8324c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:50,748 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:50,748 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., storeName=67de1f67a4e12320e34a07470b17ec6e/C, priority=12, startTime=1732137349871; duration=0sec 2024-11-20T21:15:50,748 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:50,748 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 67de1f67a4e12320e34a07470b17ec6e:C 2024-11-20T21:15:50,757 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=694, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/51048b5d1df74c6882611c3851fc9f71 2024-11-20T21:15:50,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/978d1c9bf5fc462d94ee3382e8e34460 is 50, key is test_row_0/B:col10/1732137349405/Put/seqid=0 2024-11-20T21:15:50,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742314_1490 (size=12301) 2024-11-20T21:15:51,165 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=694 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/978d1c9bf5fc462d94ee3382e8e34460 2024-11-20T21:15:51,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/6bff6527b3f74056988e8e5539530058 is 50, key is test_row_0/C:col10/1732137349405/Put/seqid=0 2024-11-20T21:15:51,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742315_1491 (size=12301) 2024-11-20T21:15:51,181 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=694 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/6bff6527b3f74056988e8e5539530058 2024-11-20T21:15:51,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/51048b5d1df74c6882611c3851fc9f71 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/51048b5d1df74c6882611c3851fc9f71 2024-11-20T21:15:51,187 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/51048b5d1df74c6882611c3851fc9f71, entries=150, sequenceid=694, filesize=30.5 K 2024-11-20T21:15:51,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/978d1c9bf5fc462d94ee3382e8e34460 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/978d1c9bf5fc462d94ee3382e8e34460 2024-11-20T21:15:51,190 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/978d1c9bf5fc462d94ee3382e8e34460, entries=150, sequenceid=694, filesize=12.0 K 2024-11-20T21:15:51,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/6bff6527b3f74056988e8e5539530058 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/6bff6527b3f74056988e8e5539530058 2024-11-20T21:15:51,194 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/6bff6527b3f74056988e8e5539530058, entries=150, sequenceid=694, filesize=12.0 K 2024-11-20T21:15:51,194 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=33.54 KB/34350 for 67de1f67a4e12320e34a07470b17ec6e in 1286ms, sequenceid=694, compaction requested=false 2024-11-20T21:15:51,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 67de1f67a4e12320e34a07470b17ec6e: 2024-11-20T21:15:51,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:51,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-20T21:15:51,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-20T21:15:51,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-20T21:15:51,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9250 sec 2024-11-20T21:15:51,200 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.9310 sec 2024-11-20T21:15:51,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-20T21:15:51,417 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:15:51,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T21:15:51,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 122 2024-11-20T21:15:51,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 115 2024-11-20T21:15:51,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-11-20T21:15:51,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 184 2024-11-20T21:15:51,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-11-20T21:15:51,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T21:15:51,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T21:15:51,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 883 2024-11-20T21:15:51,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2649 rows 2024-11-20T21:15:51,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 888 2024-11-20T21:15:51,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2664 rows 2024-11-20T21:15:51,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 885 2024-11-20T21:15:51,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2655 rows 2024-11-20T21:15:51,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 874 2024-11-20T21:15:51,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2622 rows 2024-11-20T21:15:51,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 886 2024-11-20T21:15:51,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2658 rows 2024-11-20T21:15:51,418 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T21:15:51,418 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T21:15:51,418 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:51,418 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:51,418 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:15:51,419 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-20T21:15:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T21:15:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T21:15:51,421 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732137351421"}]},"ts":"1732137351421"} 2024-11-20T21:15:51,422 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T21:15:51,422 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T21:15:51,423 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T21:15:51,424 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=67de1f67a4e12320e34a07470b17ec6e, UNASSIGN}] 2024-11-20T21:15:51,425 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=67de1f67a4e12320e34a07470b17ec6e, UNASSIGN 2024-11-20T21:15:51,425 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=67de1f67a4e12320e34a07470b17ec6e, regionState=CLOSING, regionLocation=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:51,426 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=67de1f67a4e12320e34a07470b17ec6e, UNASSIGN because future has completed 2024-11-20T21:15:51,426 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T21:15:51,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536}] 2024-11-20T21:15:51,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T21:15:51,578 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(122): Close 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:51,578 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T21:15:51,578 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1722): Closing 67de1f67a4e12320e34a07470b17ec6e, disabling compactions & flushes 2024-11-20T21:15:51,578 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:51,578 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:51,578 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. after waiting 0 ms 2024-11-20T21:15:51,578 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:51,578 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(2902): Flushing 67de1f67a4e12320e34a07470b17ec6e 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-20T21:15:51,579 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=A 2024-11-20T21:15:51,579 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:51,579 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=B 2024-11-20T21:15:51,579 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:51,579 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 67de1f67a4e12320e34a07470b17ec6e, store=C 2024-11-20T21:15:51,579 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:51,583 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200bace7f72033490896e5e80871b9f5c5_67de1f67a4e12320e34a07470b17ec6e is 50, key is test_row_0/A:col10/1732137350068/Put/seqid=0 2024-11-20T21:15:51,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742316_1492 (size=12454) 2024-11-20T21:15:51,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T21:15:51,987 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:51,991 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200bace7f72033490896e5e80871b9f5c5_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200bace7f72033490896e5e80871b9f5c5_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:51,992 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/762047a9e53d4c1ab38208f3fb41e0fe, store: [table=TestAcidGuarantees family=A region=67de1f67a4e12320e34a07470b17ec6e] 2024-11-20T21:15:51,993 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/762047a9e53d4c1ab38208f3fb41e0fe is 175, key is test_row_0/A:col10/1732137350068/Put/seqid=0 2024-11-20T21:15:51,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742317_1493 (size=31255) 2024-11-20T21:15:51,997 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=705, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/762047a9e53d4c1ab38208f3fb41e0fe 2024-11-20T21:15:52,004 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/4c76fb8c1972479087f6bd76fe3b2720 is 50, key is test_row_0/B:col10/1732137350068/Put/seqid=0 2024-11-20T21:15:52,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742318_1494 (size=12301) 2024-11-20T21:15:52,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T21:15:52,408 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=705 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/4c76fb8c1972479087f6bd76fe3b2720 2024-11-20T21:15:52,414 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/bb85f4e7b32c4505bb2876eaac4c9407 is 50, key is test_row_0/C:col10/1732137350068/Put/seqid=0 2024-11-20T21:15:52,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742319_1495 (size=12301) 2024-11-20T21:15:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T21:15:52,822 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=705 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/bb85f4e7b32c4505bb2876eaac4c9407 2024-11-20T21:15:52,826 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/A/762047a9e53d4c1ab38208f3fb41e0fe as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/762047a9e53d4c1ab38208f3fb41e0fe 2024-11-20T21:15:52,829 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/762047a9e53d4c1ab38208f3fb41e0fe, entries=150, sequenceid=705, filesize=30.5 K 2024-11-20T21:15:52,830 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/B/4c76fb8c1972479087f6bd76fe3b2720 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/4c76fb8c1972479087f6bd76fe3b2720 2024-11-20T21:15:52,833 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/4c76fb8c1972479087f6bd76fe3b2720, entries=150, sequenceid=705, filesize=12.0 K 2024-11-20T21:15:52,833 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/.tmp/C/bb85f4e7b32c4505bb2876eaac4c9407 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/bb85f4e7b32c4505bb2876eaac4c9407 2024-11-20T21:15:52,836 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/bb85f4e7b32c4505bb2876eaac4c9407, entries=150, sequenceid=705, filesize=12.0 K 2024-11-20T21:15:52,837 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 67de1f67a4e12320e34a07470b17ec6e in 1259ms, sequenceid=705, compaction requested=true 2024-11-20T21:15:52,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/768acde1c2614807a349c3d0c8a7e9dd, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/b337c51257e64dbabd76e7b164fb8a14, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/80620780a16d4204b9e302f8e82d5a48, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/b0ff3800e51043fb83ab23563e894830, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/d2336b8e0da3455382f9386c1765db15, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/f80232103adf4203a10fdd523b01f93c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/4cfdcc552c2446ac8c1779993852430e, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/2c385662618a4bb0ab60dde891b47716, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/8e0ba7e180a0457ea9e37c6b89becfcf, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/101e8f648f0c42ea8fdaf27d54d78787, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/89c270f02e944546ad67c08daaa5bc60, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/b7b6ae19ba254481bc4d6a44a0d8cbce, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/ad889b4625fa405f8976bd264400540f, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/ccacf18537da4821947bfdf245017eed, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/0c5061cd15a74e589919bce268988fd2, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/6ebaeb159b3d45bda0ca35ffc8604499, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/a901ffad323846d1ab73b84b8697f2c7, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/9b6e3c03bf2143cfaf82b59b81ecf48a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/ede2af88356448f6aac661a6a63ad109, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/819fde83a74e4f36908393f193dd3a15, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/6c33431948b049bfa57aae52f6522522, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/a480ba0999c44c928fc6692f162c3236, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/3250b45f058a41abba8671ec4cb4cae0, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/06897bfc52854e33959306a0a05387dc] to archive 2024-11-20T21:15:52,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T21:15:52,840 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/768acde1c2614807a349c3d0c8a7e9dd to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/768acde1c2614807a349c3d0c8a7e9dd 2024-11-20T21:15:52,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/b337c51257e64dbabd76e7b164fb8a14 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/b337c51257e64dbabd76e7b164fb8a14 2024-11-20T21:15:52,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/80620780a16d4204b9e302f8e82d5a48 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/80620780a16d4204b9e302f8e82d5a48 2024-11-20T21:15:52,842 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/b0ff3800e51043fb83ab23563e894830 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/b0ff3800e51043fb83ab23563e894830 2024-11-20T21:15:52,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/d2336b8e0da3455382f9386c1765db15 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/d2336b8e0da3455382f9386c1765db15 2024-11-20T21:15:52,844 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/f80232103adf4203a10fdd523b01f93c to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/f80232103adf4203a10fdd523b01f93c 2024-11-20T21:15:52,845 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/4cfdcc552c2446ac8c1779993852430e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/4cfdcc552c2446ac8c1779993852430e 2024-11-20T21:15:52,846 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/2c385662618a4bb0ab60dde891b47716 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/2c385662618a4bb0ab60dde891b47716 2024-11-20T21:15:52,846 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/8e0ba7e180a0457ea9e37c6b89becfcf to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/8e0ba7e180a0457ea9e37c6b89becfcf 2024-11-20T21:15:52,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/101e8f648f0c42ea8fdaf27d54d78787 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/101e8f648f0c42ea8fdaf27d54d78787 2024-11-20T21:15:52,848 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/89c270f02e944546ad67c08daaa5bc60 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/89c270f02e944546ad67c08daaa5bc60 2024-11-20T21:15:52,849 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/b7b6ae19ba254481bc4d6a44a0d8cbce to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/b7b6ae19ba254481bc4d6a44a0d8cbce 2024-11-20T21:15:52,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/ad889b4625fa405f8976bd264400540f to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/ad889b4625fa405f8976bd264400540f 2024-11-20T21:15:52,851 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/ccacf18537da4821947bfdf245017eed to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/ccacf18537da4821947bfdf245017eed 2024-11-20T21:15:52,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/0c5061cd15a74e589919bce268988fd2 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/0c5061cd15a74e589919bce268988fd2 2024-11-20T21:15:52,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/6ebaeb159b3d45bda0ca35ffc8604499 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/6ebaeb159b3d45bda0ca35ffc8604499 2024-11-20T21:15:52,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/a901ffad323846d1ab73b84b8697f2c7 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/a901ffad323846d1ab73b84b8697f2c7 2024-11-20T21:15:52,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/9b6e3c03bf2143cfaf82b59b81ecf48a to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/9b6e3c03bf2143cfaf82b59b81ecf48a 2024-11-20T21:15:52,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/ede2af88356448f6aac661a6a63ad109 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/ede2af88356448f6aac661a6a63ad109 2024-11-20T21:15:52,856 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/819fde83a74e4f36908393f193dd3a15 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/819fde83a74e4f36908393f193dd3a15 2024-11-20T21:15:52,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/6c33431948b049bfa57aae52f6522522 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/6c33431948b049bfa57aae52f6522522 2024-11-20T21:15:52,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/a480ba0999c44c928fc6692f162c3236 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/a480ba0999c44c928fc6692f162c3236 2024-11-20T21:15:52,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/3250b45f058a41abba8671ec4cb4cae0 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/3250b45f058a41abba8671ec4cb4cae0 2024-11-20T21:15:52,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/06897bfc52854e33959306a0a05387dc to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/06897bfc52854e33959306a0a05387dc 2024-11-20T21:15:52,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/c5bcba740de64ca9b2f7418d46b8abb6, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/c9a882036699472da7528dcf0e609dcb, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/b6f235f9d02048308f088b9ca11aa4d3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/edbcf43ee1aa4c1ca4ad632659dba5c0, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/75d6344b34bf4bdfb0c00267488c1a92, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/cd2d93259717447fa1fe8171c49f2d57, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/c4d32ad2df2a4d5cb191242c5cb0df14, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/12cecd95b61846fc9d2ba6b94583d97d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/4cd298e128ac464aba96ddbc164c4953, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/3f6952793a524c648d56972e42696acf, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/f1a569c9b31743b4a3ca2f9cc3f14082, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/2dfb0dd397a84bb2972306cfcc2cef6f, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/1db4b67f65d94d5fa871f2cf96dbd858, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/3703b4d49bb341f48801a8879b699176, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/9382302d2c1b4381a0db8de3ea189d4f, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/b12c6b06ef4f4551ba4f7ad986601699, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/2187fc64e2e04f17abebb9f75378ee3b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/b7bcf29992334ef0ba00f9bef9af7e4a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/72613ae97e57446582eaba1f4635ded9, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/ea225361d3334ee8930e58e471889c8b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/9d95a19e4dc04339868cc7a41dbc86cc, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/645d782495b6400ab2899251071e702e, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/18ad840e329347e390ee250747e62b1a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/4e966c9a107240d295333c583b9c6fe9] to archive 2024-11-20T21:15:52,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T21:15:52,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/c5bcba740de64ca9b2f7418d46b8abb6 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/c5bcba740de64ca9b2f7418d46b8abb6 2024-11-20T21:15:52,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/c9a882036699472da7528dcf0e609dcb to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/c9a882036699472da7528dcf0e609dcb 2024-11-20T21:15:52,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/b6f235f9d02048308f088b9ca11aa4d3 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/b6f235f9d02048308f088b9ca11aa4d3 2024-11-20T21:15:52,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/edbcf43ee1aa4c1ca4ad632659dba5c0 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/edbcf43ee1aa4c1ca4ad632659dba5c0 2024-11-20T21:15:52,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/75d6344b34bf4bdfb0c00267488c1a92 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/75d6344b34bf4bdfb0c00267488c1a92 2024-11-20T21:15:52,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/cd2d93259717447fa1fe8171c49f2d57 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/cd2d93259717447fa1fe8171c49f2d57 2024-11-20T21:15:52,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/c4d32ad2df2a4d5cb191242c5cb0df14 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/c4d32ad2df2a4d5cb191242c5cb0df14 2024-11-20T21:15:52,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/12cecd95b61846fc9d2ba6b94583d97d to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/12cecd95b61846fc9d2ba6b94583d97d 2024-11-20T21:15:52,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/4cd298e128ac464aba96ddbc164c4953 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/4cd298e128ac464aba96ddbc164c4953 2024-11-20T21:15:52,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/3f6952793a524c648d56972e42696acf to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/3f6952793a524c648d56972e42696acf 2024-11-20T21:15:52,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/f1a569c9b31743b4a3ca2f9cc3f14082 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/f1a569c9b31743b4a3ca2f9cc3f14082 2024-11-20T21:15:52,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/2dfb0dd397a84bb2972306cfcc2cef6f to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/2dfb0dd397a84bb2972306cfcc2cef6f 2024-11-20T21:15:52,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/1db4b67f65d94d5fa871f2cf96dbd858 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/1db4b67f65d94d5fa871f2cf96dbd858 2024-11-20T21:15:52,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/3703b4d49bb341f48801a8879b699176 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/3703b4d49bb341f48801a8879b699176 2024-11-20T21:15:52,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/9382302d2c1b4381a0db8de3ea189d4f to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/9382302d2c1b4381a0db8de3ea189d4f 2024-11-20T21:15:52,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/b12c6b06ef4f4551ba4f7ad986601699 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/b12c6b06ef4f4551ba4f7ad986601699 2024-11-20T21:15:52,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/2187fc64e2e04f17abebb9f75378ee3b to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/2187fc64e2e04f17abebb9f75378ee3b 2024-11-20T21:15:52,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/b7bcf29992334ef0ba00f9bef9af7e4a to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/b7bcf29992334ef0ba00f9bef9af7e4a 2024-11-20T21:15:52,875 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/72613ae97e57446582eaba1f4635ded9 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/72613ae97e57446582eaba1f4635ded9 2024-11-20T21:15:52,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/ea225361d3334ee8930e58e471889c8b to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/ea225361d3334ee8930e58e471889c8b 2024-11-20T21:15:52,876 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/9d95a19e4dc04339868cc7a41dbc86cc to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/9d95a19e4dc04339868cc7a41dbc86cc 2024-11-20T21:15:52,877 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/645d782495b6400ab2899251071e702e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/645d782495b6400ab2899251071e702e 2024-11-20T21:15:52,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/18ad840e329347e390ee250747e62b1a to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/18ad840e329347e390ee250747e62b1a 2024-11-20T21:15:52,878 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/4e966c9a107240d295333c583b9c6fe9 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/4e966c9a107240d295333c583b9c6fe9 2024-11-20T21:15:52,879 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/013ebcf77ba64a7180e5af1aeae21afe, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/3ca4540d44d144c280b833ebdc8989f8, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/cc272099b4054d789b9bb01b0621001f, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/d213b70e63504a8795f410877a640873, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/5272c4dc64664a2b86f50e4c85f9ec7a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/3630c1db28334e63a17b25a443d5fb03, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/19737f98c2784931ac944c85a11aee57, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/585a3d207633462696ea685d5df25528, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/080bf6471d0a4d788ea09836bf7bf129, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/5185a0fb9fd44d8892e25cceb86a1384, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/610158eb29c544199878a923bcf295f8, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/1cad4d43872048018d4881d844aa8a48, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/2b4382dd717c4485adbc57cf3f3ad2d2, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/e6c3f619702d4cf1bfa316785870a10e, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/85f46fe9bc954be2810c857cc017ee81, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/9103b5dc798e40b7926a0e9c72162a1a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/cd5fbeb805f6439db2872cd902785add, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/4b6816433078488fb6c1b40d3ac24a0f, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/480da8157eae439dbdcf7c17318b4571, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/58739e6837934ba699bac9864577788b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/a789f254701a4954a590751d26b9b97c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/1136c3c275aa486296376b75dd7a3fc8, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/38b65ac0e3564ef88bbc9153c213173f, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/e921ec23fad6408fa7a8ea02852b0474] to archive 2024-11-20T21:15:52,880 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T21:15:52,881 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/013ebcf77ba64a7180e5af1aeae21afe to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/013ebcf77ba64a7180e5af1aeae21afe 2024-11-20T21:15:52,882 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/3ca4540d44d144c280b833ebdc8989f8 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/3ca4540d44d144c280b833ebdc8989f8 2024-11-20T21:15:52,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/cc272099b4054d789b9bb01b0621001f to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/cc272099b4054d789b9bb01b0621001f 2024-11-20T21:15:52,883 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/d213b70e63504a8795f410877a640873 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/d213b70e63504a8795f410877a640873 2024-11-20T21:15:52,884 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/5272c4dc64664a2b86f50e4c85f9ec7a to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/5272c4dc64664a2b86f50e4c85f9ec7a 2024-11-20T21:15:52,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/3630c1db28334e63a17b25a443d5fb03 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/3630c1db28334e63a17b25a443d5fb03 2024-11-20T21:15:52,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/19737f98c2784931ac944c85a11aee57 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/19737f98c2784931ac944c85a11aee57 2024-11-20T21:15:52,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/585a3d207633462696ea685d5df25528 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/585a3d207633462696ea685d5df25528 2024-11-20T21:15:52,887 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/080bf6471d0a4d788ea09836bf7bf129 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/080bf6471d0a4d788ea09836bf7bf129 2024-11-20T21:15:52,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/5185a0fb9fd44d8892e25cceb86a1384 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/5185a0fb9fd44d8892e25cceb86a1384 2024-11-20T21:15:52,888 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/610158eb29c544199878a923bcf295f8 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/610158eb29c544199878a923bcf295f8 2024-11-20T21:15:52,889 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/1cad4d43872048018d4881d844aa8a48 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/1cad4d43872048018d4881d844aa8a48 2024-11-20T21:15:52,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/2b4382dd717c4485adbc57cf3f3ad2d2 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/2b4382dd717c4485adbc57cf3f3ad2d2 2024-11-20T21:15:52,890 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/e6c3f619702d4cf1bfa316785870a10e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/e6c3f619702d4cf1bfa316785870a10e 2024-11-20T21:15:52,891 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/85f46fe9bc954be2810c857cc017ee81 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/85f46fe9bc954be2810c857cc017ee81 2024-11-20T21:15:52,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/9103b5dc798e40b7926a0e9c72162a1a to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/9103b5dc798e40b7926a0e9c72162a1a 2024-11-20T21:15:52,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/cd5fbeb805f6439db2872cd902785add to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/cd5fbeb805f6439db2872cd902785add 2024-11-20T21:15:52,893 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/4b6816433078488fb6c1b40d3ac24a0f to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/4b6816433078488fb6c1b40d3ac24a0f 2024-11-20T21:15:52,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/480da8157eae439dbdcf7c17318b4571 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/480da8157eae439dbdcf7c17318b4571 2024-11-20T21:15:52,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/58739e6837934ba699bac9864577788b to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/58739e6837934ba699bac9864577788b 2024-11-20T21:15:52,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/a789f254701a4954a590751d26b9b97c to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/a789f254701a4954a590751d26b9b97c 2024-11-20T21:15:52,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/1136c3c275aa486296376b75dd7a3fc8 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/1136c3c275aa486296376b75dd7a3fc8 2024-11-20T21:15:52,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/38b65ac0e3564ef88bbc9153c213173f to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/38b65ac0e3564ef88bbc9153c213173f 2024-11-20T21:15:52,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/e921ec23fad6408fa7a8ea02852b0474 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/e921ec23fad6408fa7a8ea02852b0474 2024-11-20T21:15:52,900 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/recovered.edits/708.seqid, newMaxSeqId=708, maxSeqId=4 2024-11-20T21:15:52,901 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. 2024-11-20T21:15:52,901 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1676): Region close journal for 67de1f67a4e12320e34a07470b17ec6e: Waiting for close lock at 1732137351578Running coprocessor pre-close hooks at 1732137351578Disabling compacts and flushes for region at 1732137351578Disabling writes for close at 1732137351578Obtaining lock to block concurrent updates at 1732137351579 (+1 ms)Preparing flush snapshotting stores in 67de1f67a4e12320e34a07470b17ec6e at 1732137351579Finished memstore snapshotting TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732137351579Flushing stores of TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e. at 1732137351579Flushing 67de1f67a4e12320e34a07470b17ec6e/A: creating writer at 1732137351579Flushing mob file 67de1f67a4e12320e34a07470b17ec6e/A: appending metadata at 1732137351583 (+4 ms)Flushing mob file 67de1f67a4e12320e34a07470b17ec6e/A: closing flushed file at 1732137351583Flushing 67de1f67a4e12320e34a07470b17ec6e/A: appending metadata at 1732137351992 (+409 ms)Flushing 67de1f67a4e12320e34a07470b17ec6e/A: closing flushed file at 1732137351992Flushing 67de1f67a4e12320e34a07470b17ec6e/B: creating writer at 1732137352000 (+8 ms)Flushing 67de1f67a4e12320e34a07470b17ec6e/B: appending metadata at 1732137352003 (+3 ms)Flushing 67de1f67a4e12320e34a07470b17ec6e/B: closing flushed file at 1732137352003Flushing 67de1f67a4e12320e34a07470b17ec6e/C: creating writer at 1732137352411 (+408 ms)Flushing 67de1f67a4e12320e34a07470b17ec6e/C: appending metadata at 1732137352414 (+3 ms)Flushing 67de1f67a4e12320e34a07470b17ec6e/C: closing flushed file at 1732137352414Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@126b190d: reopening flushed file at 1732137352826 (+412 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69e14be6: reopening flushed file at 1732137352830 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@36c0358c: reopening flushed file at 1732137352833 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 67de1f67a4e12320e34a07470b17ec6e in 1259ms, sequenceid=705, compaction requested=true at 1732137352837 (+4 ms)Writing region close event to WAL at 1732137352898 (+61 ms)Running coprocessor post-close hooks at 1732137352901 (+3 ms)Closed at 1732137352901 2024-11-20T21:15:52,902 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(157): Closed 67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:52,903 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=67de1f67a4e12320e34a07470b17ec6e, regionState=CLOSED 2024-11-20T21:15:52,904 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure 67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 because future has completed 2024-11-20T21:15:52,906 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-20T21:15:52,906 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseRegionProcedure 67de1f67a4e12320e34a07470b17ec6e, server=0b16ab3f13b7,36987,1732137236536 in 1.4790 sec 2024-11-20T21:15:52,908 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-20T21:15:52,908 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=67de1f67a4e12320e34a07470b17ec6e, UNASSIGN in 1.4820 sec 2024-11-20T21:15:52,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-20T21:15:52,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4850 sec 2024-11-20T21:15:52,910 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732137352910"}]},"ts":"1732137352910"} 2024-11-20T21:15:52,911 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T21:15:52,911 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T21:15:52,912 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4930 sec 2024-11-20T21:15:53,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-20T21:15:53,577 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:15:53,578 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-20T21:15:53,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:15:53,580 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:15:53,580 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=115, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:15:53,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T21:15:53,584 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:53,587 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A, FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B, FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C, FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/recovered.edits] 2024-11-20T21:15:53,589 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/51048b5d1df74c6882611c3851fc9f71 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/51048b5d1df74c6882611c3851fc9f71 2024-11-20T21:15:53,590 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/652850ec50dc4c9fad5d295f8c66da96 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/652850ec50dc4c9fad5d295f8c66da96 2024-11-20T21:15:53,592 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/762047a9e53d4c1ab38208f3fb41e0fe to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/A/762047a9e53d4c1ab38208f3fb41e0fe 2024-11-20T21:15:53,594 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/4c76fb8c1972479087f6bd76fe3b2720 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/4c76fb8c1972479087f6bd76fe3b2720 2024-11-20T21:15:53,595 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/978d1c9bf5fc462d94ee3382e8e34460 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/978d1c9bf5fc462d94ee3382e8e34460 2024-11-20T21:15:53,596 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/d33bcaca66d7409bb950178842fa9cd8 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/B/d33bcaca66d7409bb950178842fa9cd8 2024-11-20T21:15:53,599 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/6bff6527b3f74056988e8e5539530058 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/6bff6527b3f74056988e8e5539530058 2024-11-20T21:15:53,600 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/806d5730038e4849b5e97a6d55e8324c to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/806d5730038e4849b5e97a6d55e8324c 2024-11-20T21:15:53,601 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/bb85f4e7b32c4505bb2876eaac4c9407 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/C/bb85f4e7b32c4505bb2876eaac4c9407 2024-11-20T21:15:53,603 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/recovered.edits/708.seqid to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e/recovered.edits/708.seqid 2024-11-20T21:15:53,603 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:53,604 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T21:15:53,604 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T21:15:53,605 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-20T21:15:53,608 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200b5c1e57ac59499fbbff41255206b30f_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200b5c1e57ac59499fbbff41255206b30f_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:53,609 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200bace7f72033490896e5e80871b9f5c5_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200bace7f72033490896e5e80871b9f5c5_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:53,610 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120151ab4f2c7af4f42908ddc8a968a3b21_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120151ab4f2c7af4f42908ddc8a968a3b21_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:53,611 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201d88c164277d47f2bea8e7492165cfdc_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411201d88c164277d47f2bea8e7492165cfdc_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:53,613 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202aa814b8744542c39f9b048054fa2079_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202aa814b8744542c39f9b048054fa2079_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:53,614 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202dc1eeba990f4216843877a476ba08ae_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411202dc1eeba990f4216843877a476ba08ae_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:53,615 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205eb663fc687c4c42adf2f820d63c192a_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205eb663fc687c4c42adf2f820d63c192a_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:53,617 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112066340f11376244eba3f3d42c4a79ae78_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112066340f11376244eba3f3d42c4a79ae78_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:53,618 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206960774cccf447fcb647dcbeddbb1966_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206960774cccf447fcb647dcbeddbb1966_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:53,619 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120740d9d15530e4e88993fb921b26e0d90_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120740d9d15530e4e88993fb921b26e0d90_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:53,620 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120818929234563477896266e45c07bc25c_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120818929234563477896266e45c07bc25c_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:53,622 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208345a22ffd5645f4b945152fdada76dc_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411208345a22ffd5645f4b945152fdada76dc_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:53,623 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b275b351be7b4f0b8c591ff48c49b946_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b275b351be7b4f0b8c591ff48c49b946_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:53,624 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b43e2d56a5e642e68cd6df84d440be3b_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b43e2d56a5e642e68cd6df84d440be3b_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:53,625 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120cf72a21e66664db4ac00f26b18e88a91_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120cf72a21e66664db4ac00f26b18e88a91_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:53,626 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d07156e655d54f43bf5fdad83e85fbf8_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120d07156e655d54f43bf5fdad83e85fbf8_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:53,628 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e30f64eaca034b4ab25560246d3d17d2_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e30f64eaca034b4ab25560246d3d17d2_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:53,629 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120efec5a7030ae43cda6bd5ec0cba95d3c_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120efec5a7030ae43cda6bd5ec0cba95d3c_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:53,630 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f70ad28cccff444d9465811ea56641ca_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f70ad28cccff444d9465811ea56641ca_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:53,631 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f97586f3529549beac2f0d3e0c0d2a52_67de1f67a4e12320e34a07470b17ec6e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120f97586f3529549beac2f0d3e0c0d2a52_67de1f67a4e12320e34a07470b17ec6e 2024-11-20T21:15:53,631 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T21:15:53,634 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=115, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:15:53,637 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T21:15:53,639 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T21:15:53,640 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=115, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:15:53,640 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T21:15:53,640 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732137353640"}]},"ts":"9223372036854775807"} 2024-11-20T21:15:53,643 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T21:15:53,643 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 67de1f67a4e12320e34a07470b17ec6e, NAME => 'TestAcidGuarantees,,1732137327057.67de1f67a4e12320e34a07470b17ec6e.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T21:15:53,643 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T21:15:53,643 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732137353643"}]},"ts":"9223372036854775807"} 2024-11-20T21:15:53,645 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T21:15:53,650 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=115, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:15:53,651 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 72 msec 2024-11-20T21:15:53,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-20T21:15:53,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T21:15:53,687 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:15:53,696 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=249 (was 247) - Thread LEAK? -, OpenFileDescriptor=453 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=809 (was 833), ProcessCount=11 (was 11), AvailableMemoryMB=998 (was 824) - AvailableMemoryMB LEAK? - 2024-11-20T21:15:53,704 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=249, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=809, ProcessCount=11, AvailableMemoryMB=998 2024-11-20T21:15:53,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T21:15:53,705 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T21:15:53,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T21:15:53,707 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T21:15:53,707 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:53,707 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 116 2024-11-20T21:15:53,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-20T21:15:53,708 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T21:15:53,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742320_1496 (size=1061) 2024-11-20T21:15:53,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-20T21:15:54,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-20T21:15:54,114 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094 2024-11-20T21:15:54,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742321_1497 (size=53) 2024-11-20T21:15:54,120 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:15:54,120 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6c66f98c982744caf390af289e0ec6cf, disabling compactions & flushes 2024-11-20T21:15:54,120 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:15:54,120 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:15:54,120 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. after waiting 0 ms 2024-11-20T21:15:54,120 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:15:54,120 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:15:54,120 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6c66f98c982744caf390af289e0ec6cf: Waiting for close lock at 1732137354120Disabling compacts and flushes for region at 1732137354120Disabling writes for close at 1732137354120Writing region close event to WAL at 1732137354120Closed at 1732137354120 2024-11-20T21:15:54,121 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T21:15:54,122 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732137354121"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732137354121"}]},"ts":"1732137354121"} 2024-11-20T21:15:54,123 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T21:15:54,124 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T21:15:54,124 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732137354124"}]},"ts":"1732137354124"} 2024-11-20T21:15:54,125 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T21:15:54,125 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c66f98c982744caf390af289e0ec6cf, ASSIGN}] 2024-11-20T21:15:54,126 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c66f98c982744caf390af289e0ec6cf, ASSIGN 2024-11-20T21:15:54,127 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c66f98c982744caf390af289e0ec6cf, ASSIGN; state=OFFLINE, location=0b16ab3f13b7,36987,1732137236536; forceNewPlan=false, retain=false 2024-11-20T21:15:54,277 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=6c66f98c982744caf390af289e0ec6cf, regionState=OPENING, regionLocation=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:54,279 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c66f98c982744caf390af289e0ec6cf, ASSIGN because future has completed 2024-11-20T21:15:54,279 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536}] 2024-11-20T21:15:54,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-20T21:15:54,434 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:15:54,434 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7752): Opening region: {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:15:54,435 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:15:54,435 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:15:54,436 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7794): checking encryption for 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:15:54,436 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7797): checking classloading for 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:15:54,437 INFO [StoreOpener-6c66f98c982744caf390af289e0ec6cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:15:54,438 INFO [StoreOpener-6c66f98c982744caf390af289e0ec6cf-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T21:15:54,438 INFO [StoreOpener-6c66f98c982744caf390af289e0ec6cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c66f98c982744caf390af289e0ec6cf columnFamilyName A 2024-11-20T21:15:54,438 DEBUG [StoreOpener-6c66f98c982744caf390af289e0ec6cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:54,439 INFO [StoreOpener-6c66f98c982744caf390af289e0ec6cf-1 {}] regionserver.HStore(327): Store=6c66f98c982744caf390af289e0ec6cf/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:15:54,439 INFO [StoreOpener-6c66f98c982744caf390af289e0ec6cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:15:54,440 INFO [StoreOpener-6c66f98c982744caf390af289e0ec6cf-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T21:15:54,440 INFO [StoreOpener-6c66f98c982744caf390af289e0ec6cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c66f98c982744caf390af289e0ec6cf columnFamilyName B 2024-11-20T21:15:54,440 DEBUG [StoreOpener-6c66f98c982744caf390af289e0ec6cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:54,441 INFO [StoreOpener-6c66f98c982744caf390af289e0ec6cf-1 {}] regionserver.HStore(327): Store=6c66f98c982744caf390af289e0ec6cf/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:15:54,441 INFO [StoreOpener-6c66f98c982744caf390af289e0ec6cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:15:54,442 INFO [StoreOpener-6c66f98c982744caf390af289e0ec6cf-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T21:15:54,442 INFO [StoreOpener-6c66f98c982744caf390af289e0ec6cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c66f98c982744caf390af289e0ec6cf columnFamilyName C 2024-11-20T21:15:54,442 DEBUG [StoreOpener-6c66f98c982744caf390af289e0ec6cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:15:54,443 INFO [StoreOpener-6c66f98c982744caf390af289e0ec6cf-1 {}] regionserver.HStore(327): Store=6c66f98c982744caf390af289e0ec6cf/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:15:54,443 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:15:54,443 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1038): replaying wal for 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:15:54,443 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:15:54,444 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:15:54,445 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1048): stopping wal replay for 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:15:54,445 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1060): Cleaning up temporary data for 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:15:54,446 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T21:15:54,447 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1093): writing seq id for 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:15:54,449 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T21:15:54,450 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1114): Opened 6c66f98c982744caf390af289e0ec6cf; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71056123, jitterRate=0.058818742632865906}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T21:15:54,450 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:15:54,451 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1006): Region open journal for 6c66f98c982744caf390af289e0ec6cf: Running coprocessor pre-open hook at 1732137354436Writing region info on filesystem at 1732137354436Initializing all the Stores at 1732137354436Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732137354436Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732137354437 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732137354437Cleaning up temporary data from old regions at 1732137354445 (+8 ms)Running coprocessor post-open hooks at 1732137354450 (+5 ms)Region opened successfully at 1732137354451 (+1 ms) 2024-11-20T21:15:54,452 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., pid=118, masterSystemTime=1732137354431 2024-11-20T21:15:54,454 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:15:54,454 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:15:54,454 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=6c66f98c982744caf390af289e0ec6cf, regionState=OPEN, openSeqNum=2, regionLocation=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:54,456 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 because future has completed 2024-11-20T21:15:54,459 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-20T21:15:54,459 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; OpenRegionProcedure 6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 in 178 msec 2024-11-20T21:15:54,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-20T21:15:54,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c66f98c982744caf390af289e0ec6cf, ASSIGN in 334 msec 2024-11-20T21:15:54,462 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T21:15:54,462 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732137354462"}]},"ts":"1732137354462"} 2024-11-20T21:15:54,463 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T21:15:54,464 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T21:15:54,466 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 759 msec 2024-11-20T21:15:54,815 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T21:15:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-20T21:15:54,847 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:15:54,847 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3005a7e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:54,848 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:15:54,848 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:15:54,849 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:15:54,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:15:54,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:15:54,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@756b73ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:54,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:15:54,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:15:54,850 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:54,850 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:15:54,851 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@376a4c28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:54,851 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:15:54,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:15:54,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:15:54,853 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:15:54,854 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T21:15:54,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 0b16ab3f13b7,39687,1732137235812 2024-11-20T21:15:54,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@778344e0 2024-11-20T21:15:54,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T21:15:54,855 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58860, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T21:15:54,856 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72417d0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:54,856 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:15:54,857 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:15:54,857 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:15:54,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:15:54,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:15:54,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@348a6dd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:54,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:15:54,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:15:54,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:54,858 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:15:54,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2566d302, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:54,859 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@581c4bd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:54,859 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:15:54,859 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:15:54,860 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:15:54,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:15:54,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:15:54,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3136857e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:54,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:15:54,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:15:54,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:54,861 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:15:54,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a2da8aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:54,862 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3692791b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:54,862 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:15:54,862 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:15:54,862 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:15:54,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:15:54,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:15:54,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a24af57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:54,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:15:54,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:15:54,863 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:54,863 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:15:54,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43d9e962, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:54,864 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b791feb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:54,864 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:15:54,864 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:15:54,865 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:15:54,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:15:54,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:15:54,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d241d2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:54,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:15:54,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:15:54,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:54,866 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58924, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:15:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61e05bb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:54,867 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da2e0be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:54,867 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:15:54,867 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:15:54,868 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:15:54,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:15:54,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:15:54,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4637797a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:54,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:15:54,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:15:54,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:54,869 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:15:54,869 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b7a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:54,870 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2822c33e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:54,870 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:15:54,870 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:15:54,870 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:15:54,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:15:54,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:15:54,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@346fa785, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:54,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:15:54,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:15:54,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:54,872 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:15:54,872 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f226544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:54,872 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f59c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:54,873 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:15:54,873 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:15:54,873 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:15:54,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:15:54,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:15:54,873 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@420a441, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:54,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:15:54,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:15:54,874 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:54,874 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:15:54,875 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75641b22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:54,875 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ae789b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:54,876 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:15:54,876 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:15:54,876 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:15:54,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:15:54,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:15:54,877 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d0ba117, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:54,877 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:15:54,877 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:15:54,877 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:54,877 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:15:54,878 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e57e6bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:54,878 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b7b3b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:54,878 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:15:54,879 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:15:54,879 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:15:54,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:15:54,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:15:54,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@202c33f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:54,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:15:54,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:15:54,880 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:54,880 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:15:54,881 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30113130, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:54,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e04f49c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:54,881 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:15:54,881 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:15:54,882 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:15:54,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:15:54,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:15:54,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33df841b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:54,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:15:54,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:15:54,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:15:54,883 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59020, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:15:54,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a3ba456, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:15:54,884 DEBUG [Thread-2215 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:15:54,884 DEBUG [Thread-2202 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:15:54,885 DEBUG [Thread-2206 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:15:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:15:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:15:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:15:54,885 DEBUG [Thread-2219 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:15:54,885 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:15:54,886 DEBUG [Thread-2204 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:15:54,886 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:15:54,886 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:15:54,886 DEBUG [Thread-2213 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:15:54,886 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:15:54,886 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:15:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:15:54,887 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:15:54,887 DEBUG [Thread-2221 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:15:54,887 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:15:54,887 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:15:54,887 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:15:54,887 DEBUG [Thread-2210 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:15:54,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:15:54,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:15:54,888 DEBUG [Thread-2208 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:15:54,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2] 2024-11-20T21:15:54,888 DEBUG [Thread-2217 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:15:54,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:15:54,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:15:54,888 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:15:54,888 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:15:54,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:15:54,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:15:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:15:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:15:54,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2] 2024-11-20T21:15:54,890 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:15:54,890 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:15:54,890 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:15:54,890 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:15:54,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:15:54,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2] 2024-11-20T21:15:54,891 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:15:54,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2] 2024-11-20T21:15:54,891 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2] 2024-11-20T21:15:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-20T21:15:54,892 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2] 2024-11-20T21:15:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T21:15:54,893 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2] 2024-11-20T21:15:54,893 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:15:54,893 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:15:54,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2] 2024-11-20T21:15:54,894 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:15:54,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:15:54,895 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2] 2024-11-20T21:15:54,897 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:15:54,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2] 2024-11-20T21:15:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:15:54,899 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c66f98c982744caf390af289e0ec6cf 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T21:15:54,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=A 2024-11-20T21:15:54,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:54,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=B 2024-11-20T21:15:54,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:54,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=C 2024-11-20T21:15:54,900 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:54,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/ff18c5f207b04f43bf85c605f7d4004f is 50, key is test_row_0/A:col10/1732137354898/Put/seqid=0 2024-11-20T21:15:54,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742322_1498 (size=16681) 2024-11-20T21:15:54,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1732137354932, max timestamp=1732137354949Num uniques 859; 2024-11-20T21:15:54,956 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T21:15:54,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732137354932, max timestamp=1732137354955Num uniques 859; 2024-11-20T21:15:54,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T21:15:54,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732137354932, max timestamp=1732137354949Num uniques 859; 2024-11-20T21:15:54,959 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T21:15:55,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T21:15:55,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:55,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137414997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:55,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:55,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137415001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:55,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:55,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137415002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137415003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:55,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137415006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,020 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,021 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:55,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,021 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:55,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:55,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-20T21:15:55,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:15:55,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:15:55,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:15:55,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:55,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:55,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:55,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:55,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137415126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:55,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:55,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:55,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137415126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:55,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:55,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137415126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:55,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:55,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:55,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137415126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:55,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:55,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:55,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137415127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:55,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,138 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:55,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:55,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T21:15:55,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-20T21:15:55,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:15:55,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:15:55,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:15:55,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:55,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:55,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:55,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/ff18c5f207b04f43bf85c605f7d4004f 2024-11-20T21:15:55,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:55,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137415347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:55,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,363 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:55,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-20T21:15:55,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:15:55,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:15:55,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:15:55,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:55,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:55,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:55,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:55,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137415348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:55,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137415348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:55,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:55,365 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:55,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137415348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:55,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:55,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:55,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137415348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:55,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:55,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:55,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:55,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/97fdeea592f94ff18f7fa19efbceb48a is 50, key is test_row_0/B:col10/1732137354898/Put/seqid=0 2024-11-20T21:15:55,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742323_1499 (size=12001) 2024-11-20T21:15:55,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T21:15:55,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-20T21:15:55,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:15:55,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:15:55,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:15:55,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:55,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:55,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137415667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:55,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137415667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:55,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:55,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:55,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-20T21:15:55,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:15:55,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:15:55,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:15:55,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:55,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:55,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:55,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137415676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:55,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:55,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137415677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:55,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:55,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:55,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137415677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:55,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:55,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:55,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:55,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-20T21:15:55,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:15:55,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:15:55,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:15:55,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/97fdeea592f94ff18f7fa19efbceb48a 2024-11-20T21:15:55,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:55,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:55,859 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/5774c18734e046f29cfae329c10155a7 is 50, key is test_row_0/C:col10/1732137354898/Put/seqid=0 2024-11-20T21:15:55,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742324_1500 (size=12001) 2024-11-20T21:15:55,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/5774c18734e046f29cfae329c10155a7 2024-11-20T21:15:55,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/ff18c5f207b04f43bf85c605f7d4004f as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/ff18c5f207b04f43bf85c605f7d4004f 2024-11-20T21:15:55,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/ff18c5f207b04f43bf85c605f7d4004f, entries=250, sequenceid=11, filesize=16.3 K 2024-11-20T21:15:55,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/97fdeea592f94ff18f7fa19efbceb48a as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/97fdeea592f94ff18f7fa19efbceb48a 2024-11-20T21:15:55,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/97fdeea592f94ff18f7fa19efbceb48a, entries=150, sequenceid=11, filesize=11.7 K 2024-11-20T21:15:55,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/5774c18734e046f29cfae329c10155a7 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/5774c18734e046f29cfae329c10155a7 2024-11-20T21:15:55,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/5774c18734e046f29cfae329c10155a7, entries=150, sequenceid=11, filesize=11.7 K 2024-11-20T21:15:55,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 6c66f98c982744caf390af289e0ec6cf in 983ms, sequenceid=11, compaction requested=false 2024-11-20T21:15:55,882 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-20T21:15:55,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:15:55,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-20T21:15:55,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:15:55,990 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 6c66f98c982744caf390af289e0ec6cf 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-20T21:15:55,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=A 2024-11-20T21:15:55,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:55,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=B 2024-11-20T21:15:55,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:55,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=C 2024-11-20T21:15:55,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:56,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/eaf04f4234464b1c862bdad3f88ad015 is 50, key is test_row_0/A:col10/1732137354997/Put/seqid=0 2024-11-20T21:15:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T21:15:56,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742325_1501 (size=12001) 2024-11-20T21:15:56,050 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/eaf04f4234464b1c862bdad3f88ad015 2024-11-20T21:15:56,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/9cb531db46e848a486b7ab087123122b is 50, key is test_row_0/B:col10/1732137354997/Put/seqid=0 2024-11-20T21:15:56,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742326_1502 (size=12001) 2024-11-20T21:15:56,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:15:56,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:15:56,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:56,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137416216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:56,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:56,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:56,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137416217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:56,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:56,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:56,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137416218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:56,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:56,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137416221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:56,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:56,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:56,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137416223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:56,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:56,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:56,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:56,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137416336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:56,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:56,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:56,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137416347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:56,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:56,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137416347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:56,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:56,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:56,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137416347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:56,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:56,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:56,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:56,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137416347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:56,359 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,359 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,359 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:56,529 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/9cb531db46e848a486b7ab087123122b 2024-11-20T21:15:56,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:56,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137416549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:56,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:56,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/a68990a92ed34c47ae4b555f0bb6091b is 50, key is test_row_0/C:col10/1732137354997/Put/seqid=0 2024-11-20T21:15:56,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:56,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137416569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:56,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:56,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137416569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,579 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:56,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:56,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137416570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:56,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:56,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137416570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:56,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:56,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,580 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:56,583 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,583 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,583 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:56,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742327_1503 (size=12001) 2024-11-20T21:15:56,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:56,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137416866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:56,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:56,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137416887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:56,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:56,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137416887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:56,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:56,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:56,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137416887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:56,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:56,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:56,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:56,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137416887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:56,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:56,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:57,018 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/a68990a92ed34c47ae4b555f0bb6091b 2024-11-20T21:15:57,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/eaf04f4234464b1c862bdad3f88ad015 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/eaf04f4234464b1c862bdad3f88ad015 2024-11-20T21:15:57,026 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/eaf04f4234464b1c862bdad3f88ad015, entries=150, sequenceid=47, filesize=11.7 K 2024-11-20T21:15:57,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/9cb531db46e848a486b7ab087123122b as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/9cb531db46e848a486b7ab087123122b 2024-11-20T21:15:57,029 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/9cb531db46e848a486b7ab087123122b, entries=150, sequenceid=47, filesize=11.7 K 2024-11-20T21:15:57,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/a68990a92ed34c47ae4b555f0bb6091b as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/a68990a92ed34c47ae4b555f0bb6091b 2024-11-20T21:15:57,034 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/a68990a92ed34c47ae4b555f0bb6091b, entries=150, sequenceid=47, filesize=11.7 K 2024-11-20T21:15:57,034 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 6c66f98c982744caf390af289e0ec6cf in 1045ms, sequenceid=47, compaction requested=false 2024-11-20T21:15:57,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:15:57,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:15:57,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-20T21:15:57,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-20T21:15:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T21:15:57,038 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-20T21:15:57,038 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1420 sec 2024-11-20T21:15:57,040 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.1510 sec 2024-11-20T21:15:57,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:15:57,393 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c66f98c982744caf390af289e0ec6cf 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T21:15:57,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=A 2024-11-20T21:15:57,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:57,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=B 2024-11-20T21:15:57,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:57,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=C 2024-11-20T21:15:57,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:57,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/0d1d886d81a74fb4a9e75fb97ef3f7d9 is 50, key is test_row_0/A:col10/1732137356218/Put/seqid=0 2024-11-20T21:15:57,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742328_1504 (size=14341) 2024-11-20T21:15:57,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/0d1d886d81a74fb4a9e75fb97ef3f7d9 2024-11-20T21:15:57,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/f9ae4e278d27417c84f805e741e4d326 is 50, key is test_row_0/B:col10/1732137356218/Put/seqid=0 2024-11-20T21:15:57,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742329_1505 (size=12001) 2024-11-20T21:15:57,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/f9ae4e278d27417c84f805e741e4d326 2024-11-20T21:15:57,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/d2c0a14a27ce4a869eeae57b9158b9d4 is 50, key is test_row_0/C:col10/1732137356218/Put/seqid=0 2024-11-20T21:15:57,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742330_1506 (size=12001) 2024-11-20T21:15:57,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732137357416, max timestamp=1732137357462Num uniques 859; 2024-11-20T21:15:57,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732137357416, max timestamp=1732137357462Num uniques 859; 2024-11-20T21:15:57,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T21:15:57,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T21:15:57,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732137357416, max timestamp=1732137357462Num uniques 859; 2024-11-20T21:15:57,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T21:15:57,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:57,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137417501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:57,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:57,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:57,512 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:57,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:57,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:57,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137417506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:57,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137417508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:57,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:57,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:57,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:57,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:57,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137417510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:57,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:57,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137417512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:57,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:57,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:57,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:57,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:57,533 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:57,533 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:57,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:57,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:57,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:57,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:57,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137417621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:57,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:57,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:57,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137417637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:57,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:57,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:57,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:57,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:57,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137417637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:57,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:57,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:57,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:57,655 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:57,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137417638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:57,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:57,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137417638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:57,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:57,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:57,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:57,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:57,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:57,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:57,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/d2c0a14a27ce4a869eeae57b9158b9d4 2024-11-20T21:15:57,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/0d1d886d81a74fb4a9e75fb97ef3f7d9 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/0d1d886d81a74fb4a9e75fb97ef3f7d9 2024-11-20T21:15:57,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/0d1d886d81a74fb4a9e75fb97ef3f7d9, entries=200, sequenceid=58, filesize=14.0 K 2024-11-20T21:15:57,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/f9ae4e278d27417c84f805e741e4d326 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/f9ae4e278d27417c84f805e741e4d326 2024-11-20T21:15:57,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:57,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137417846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:57,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:57,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:57,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:57,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/f9ae4e278d27417c84f805e741e4d326, entries=150, sequenceid=58, filesize=11.7 K 2024-11-20T21:15:57,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/d2c0a14a27ce4a869eeae57b9158b9d4 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/d2c0a14a27ce4a869eeae57b9158b9d4 2024-11-20T21:15:57,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/d2c0a14a27ce4a869eeae57b9158b9d4, entries=150, sequenceid=58, filesize=11.7 K 2024-11-20T21:15:57,865 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 6c66f98c982744caf390af289e0ec6cf in 472ms, sequenceid=58, compaction requested=true 2024-11-20T21:15:57,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:15:57,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c66f98c982744caf390af289e0ec6cf:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:15:57,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:57,866 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:57,866 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:57,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c66f98c982744caf390af289e0ec6cf:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:15:57,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:57,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c66f98c982744caf390af289e0ec6cf:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:15:57,866 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:57,866 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:57,867 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:57,867 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 6c66f98c982744caf390af289e0ec6cf/B is initiating minor compaction (all files) 2024-11-20T21:15:57,867 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 6c66f98c982744caf390af289e0ec6cf/A is initiating minor compaction (all files) 2024-11-20T21:15:57,867 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c66f98c982744caf390af289e0ec6cf/A in TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:15:57,867 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c66f98c982744caf390af289e0ec6cf/B in TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:15:57,867 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/ff18c5f207b04f43bf85c605f7d4004f, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/eaf04f4234464b1c862bdad3f88ad015, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/0d1d886d81a74fb4a9e75fb97ef3f7d9] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp, totalSize=42.0 K 2024-11-20T21:15:57,867 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/97fdeea592f94ff18f7fa19efbceb48a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/9cb531db46e848a486b7ab087123122b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/f9ae4e278d27417c84f805e741e4d326] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp, totalSize=35.2 K 2024-11-20T21:15:57,867 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff18c5f207b04f43bf85c605f7d4004f, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732137354893 2024-11-20T21:15:57,867 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 97fdeea592f94ff18f7fa19efbceb48a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732137354893 2024-11-20T21:15:57,867 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting eaf04f4234464b1c862bdad3f88ad015, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732137354997 2024-11-20T21:15:57,867 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cb531db46e848a486b7ab087123122b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732137354997 2024-11-20T21:15:57,868 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d1d886d81a74fb4a9e75fb97ef3f7d9, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732137356217 2024-11-20T21:15:57,868 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting f9ae4e278d27417c84f805e741e4d326, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732137356217 2024-11-20T21:15:57,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:15:57,870 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c66f98c982744caf390af289e0ec6cf 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-20T21:15:57,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=A 2024-11-20T21:15:57,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:57,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=B 2024-11-20T21:15:57,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:57,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=C 2024-11-20T21:15:57,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:57,876 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c66f98c982744caf390af289e0ec6cf#A#compaction#422 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:57,876 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/fe4c3aafc97c4cd7b69f629d26da4a23 is 50, key is test_row_0/A:col10/1732137356218/Put/seqid=0 2024-11-20T21:15:57,877 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c66f98c982744caf390af289e0ec6cf#B#compaction#423 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:57,877 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/9893e32ac671412b9e433d69c35f921b is 50, key is test_row_0/B:col10/1732137356218/Put/seqid=0 2024-11-20T21:15:57,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/35e6431dc25b4a15adccc62a7c715ac4 is 50, key is test_row_0/A:col10/1732137357868/Put/seqid=0 2024-11-20T21:15:57,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742331_1507 (size=12104) 2024-11-20T21:15:57,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742332_1508 (size=12104) 2024-11-20T21:15:57,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742333_1509 (size=14341) 2024-11-20T21:15:57,891 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/fe4c3aafc97c4cd7b69f629d26da4a23 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/fe4c3aafc97c4cd7b69f629d26da4a23 2024-11-20T21:15:57,895 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c66f98c982744caf390af289e0ec6cf/A of 6c66f98c982744caf390af289e0ec6cf into fe4c3aafc97c4cd7b69f629d26da4a23(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:57,895 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:15:57,895 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., storeName=6c66f98c982744caf390af289e0ec6cf/A, priority=13, startTime=1732137357866; duration=0sec 2024-11-20T21:15:57,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:57,895 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:57,895 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c66f98c982744caf390af289e0ec6cf:A 2024-11-20T21:15:57,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137417877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:57,895 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:57,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:57,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:57,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:57,896 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:57,896 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 6c66f98c982744caf390af289e0ec6cf/C is initiating minor compaction (all files) 2024-11-20T21:15:57,896 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c66f98c982744caf390af289e0ec6cf/C in TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:15:57,896 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/5774c18734e046f29cfae329c10155a7, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/a68990a92ed34c47ae4b555f0bb6091b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/d2c0a14a27ce4a869eeae57b9158b9d4] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp, totalSize=35.2 K 2024-11-20T21:15:57,897 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5774c18734e046f29cfae329c10155a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732137354893 2024-11-20T21:15:57,897 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting a68990a92ed34c47ae4b555f0bb6091b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1732137354997 2024-11-20T21:15:57,897 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2c0a14a27ce4a869eeae57b9158b9d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732137356217 2024-11-20T21:15:57,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:57,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137417886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:57,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:57,903 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c66f98c982744caf390af289e0ec6cf#C#compaction#425 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:57,904 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/10875b29b3ef4838b4ac181225e27b62 is 50, key is test_row_0/C:col10/1732137356218/Put/seqid=0 2024-11-20T21:15:57,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:57,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137417890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:57,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:57,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:57,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:57,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:57,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137417895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:57,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:57,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:57,908 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:57,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742334_1510 (size=12104) 2024-11-20T21:15:58,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:58,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137417996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:58,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:58,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:58,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137418007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:58,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:58,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:58,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137418017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:58,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:58,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:58,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137418017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:58,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:58,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:58,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137418166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:58,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:58,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:58,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137418217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:58,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:58,225 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:58,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137418218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:58,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:58,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:58,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137418237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:58,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:58,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:58,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137418237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:58,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:58,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/35e6431dc25b4a15adccc62a7c715ac4 2024-11-20T21:15:58,300 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/9893e32ac671412b9e433d69c35f921b as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/9893e32ac671412b9e433d69c35f921b 2024-11-20T21:15:58,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/713a1ce5d04a410baf849d281f638277 is 50, key is test_row_0/B:col10/1732137357868/Put/seqid=0 2024-11-20T21:15:58,318 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c66f98c982744caf390af289e0ec6cf/B of 6c66f98c982744caf390af289e0ec6cf into 9893e32ac671412b9e433d69c35f921b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:58,318 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:15:58,318 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., storeName=6c66f98c982744caf390af289e0ec6cf/B, priority=13, startTime=1732137357866; duration=0sec 2024-11-20T21:15:58,318 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:58,318 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c66f98c982744caf390af289e0ec6cf:B 2024-11-20T21:15:58,340 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/10875b29b3ef4838b4ac181225e27b62 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/10875b29b3ef4838b4ac181225e27b62 2024-11-20T21:15:58,345 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c66f98c982744caf390af289e0ec6cf/C of 6c66f98c982744caf390af289e0ec6cf into 10875b29b3ef4838b4ac181225e27b62(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:58,345 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:15:58,345 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., storeName=6c66f98c982744caf390af289e0ec6cf/C, priority=13, startTime=1732137357866; duration=0sec 2024-11-20T21:15:58,345 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:58,345 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c66f98c982744caf390af289e0ec6cf:C 2024-11-20T21:15:58,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742335_1511 (size=12001) 2024-11-20T21:15:58,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/713a1ce5d04a410baf849d281f638277 2024-11-20T21:15:58,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/afc83d782713497cbd9dbd2c48acf402 is 50, key is test_row_0/C:col10/1732137357868/Put/seqid=0 2024-11-20T21:15:58,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742336_1512 (size=12001) 2024-11-20T21:15:58,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/afc83d782713497cbd9dbd2c48acf402 2024-11-20T21:15:58,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/35e6431dc25b4a15adccc62a7c715ac4 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/35e6431dc25b4a15adccc62a7c715ac4 2024-11-20T21:15:58,418 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T21:15:58,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/35e6431dc25b4a15adccc62a7c715ac4, entries=200, sequenceid=94, filesize=14.0 K 2024-11-20T21:15:58,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/713a1ce5d04a410baf849d281f638277 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/713a1ce5d04a410baf849d281f638277 2024-11-20T21:15:58,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/713a1ce5d04a410baf849d281f638277, entries=150, sequenceid=94, filesize=11.7 K 2024-11-20T21:15:58,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/afc83d782713497cbd9dbd2c48acf402 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/afc83d782713497cbd9dbd2c48acf402 2024-11-20T21:15:58,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/afc83d782713497cbd9dbd2c48acf402, entries=150, sequenceid=94, filesize=11.7 K 2024-11-20T21:15:58,449 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 6c66f98c982744caf390af289e0ec6cf in 579ms, sequenceid=94, compaction requested=false 2024-11-20T21:15:58,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:15:58,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:15:58,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c66f98c982744caf390af289e0ec6cf 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T21:15:58,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=A 2024-11-20T21:15:58,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:58,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=B 2024-11-20T21:15:58,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:58,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=C 2024-11-20T21:15:58,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:15:58,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/49e3e8bfb60f41d3a8f4b7a7a5cd2784 is 50, key is test_row_0/A:col10/1732137357874/Put/seqid=0 2024-11-20T21:15:58,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742337_1513 (size=12001) 2024-11-20T21:15:58,561 WARN [regionserver/0b16ab3f13b7:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 0 2024-11-20T21:15:58,563 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b16ab3f13b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/ff18c5f207b04f43bf85c605f7d4004f, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/eaf04f4234464b1c862bdad3f88ad015, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/0d1d886d81a74fb4a9e75fb97ef3f7d9] to archive 2024-11-20T21:15:58,564 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b16ab3f13b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T21:15:58,566 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b16ab3f13b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/ff18c5f207b04f43bf85c605f7d4004f to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/ff18c5f207b04f43bf85c605f7d4004f 2024-11-20T21:15:58,569 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b16ab3f13b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/eaf04f4234464b1c862bdad3f88ad015 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/eaf04f4234464b1c862bdad3f88ad015 2024-11-20T21:15:58,570 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b16ab3f13b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/0d1d886d81a74fb4a9e75fb97ef3f7d9 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/0d1d886d81a74fb4a9e75fb97ef3f7d9 2024-11-20T21:15:58,571 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b16ab3f13b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/97fdeea592f94ff18f7fa19efbceb48a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/9cb531db46e848a486b7ab087123122b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/f9ae4e278d27417c84f805e741e4d326] to archive 2024-11-20T21:15:58,572 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b16ab3f13b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T21:15:58,576 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b16ab3f13b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/97fdeea592f94ff18f7fa19efbceb48a to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/97fdeea592f94ff18f7fa19efbceb48a 2024-11-20T21:15:58,577 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b16ab3f13b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/9cb531db46e848a486b7ab087123122b to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/9cb531db46e848a486b7ab087123122b 2024-11-20T21:15:58,579 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b16ab3f13b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/f9ae4e278d27417c84f805e741e4d326 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/f9ae4e278d27417c84f805e741e4d326 2024-11-20T21:15:58,581 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b16ab3f13b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/5774c18734e046f29cfae329c10155a7, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/a68990a92ed34c47ae4b555f0bb6091b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/d2c0a14a27ce4a869eeae57b9158b9d4] to archive 2024-11-20T21:15:58,588 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b16ab3f13b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T21:15:58,596 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b16ab3f13b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/5774c18734e046f29cfae329c10155a7 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/5774c18734e046f29cfae329c10155a7 2024-11-20T21:15:58,597 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b16ab3f13b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/a68990a92ed34c47ae4b555f0bb6091b to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/a68990a92ed34c47ae4b555f0bb6091b 2024-11-20T21:15:58,601 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/0b16ab3f13b7:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/d2c0a14a27ce4a869eeae57b9158b9d4 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/d2c0a14a27ce4a869eeae57b9158b9d4 2024-11-20T21:15:58,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732137358578, max timestamp=1732137358600Num uniques 859; 2024-11-20T21:15:58,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732137358578, max timestamp=1732137358600Num uniques 859; 2024-11-20T21:15:58,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T21:15:58,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T21:15:58,607 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732137358578, max timestamp=1732137358600Num uniques 859; 2024-11-20T21:15:58,607 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T21:15:58,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:58,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137418670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:58,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,687 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:58,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:58,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137418674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:58,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,696 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:58,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:58,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137418676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:58,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:58,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137418677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:58,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:58,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:58,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:58,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137418688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:58,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:58,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137418797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:58,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:58,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:58,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137418807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:58,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:58,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137418807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:58,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:58,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137418807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:58,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:58,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,823 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:58,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:58,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:58,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/49e3e8bfb60f41d3a8f4b7a7a5cd2784 2024-11-20T21:15:58,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/67f6b210352e406598d7f3180f7003f3 is 50, key is test_row_0/B:col10/1732137357874/Put/seqid=0 2024-11-20T21:15:58,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742338_1514 (size=12001) 2024-11-20T21:15:58,976 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/67f6b210352e406598d7f3180f7003f3 2024-11-20T21:15:58,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/766bc78b98f0445a88088b04aff78c83 is 50, key is test_row_0/C:col10/1732137357874/Put/seqid=0 2024-11-20T21:15:59,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742339_1515 (size=12001) 2024-11-20T21:15:59,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:59,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137419008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:59,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:59,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:59,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:59,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:59,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137419027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:59,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:59,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:59,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:59,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:59,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137419027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:59,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:59,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137419027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:59,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:59,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:59,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:59,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:59,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:59,041 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:59,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-20T21:15:59,047 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:15:59,047 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:15:59,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-20T21:15:59,049 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:15:59,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T21:15:59,050 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:15:59,050 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:15:59,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T21:15:59,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T21:15:59,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:15:59,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:15:59,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:15:59,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:59,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:59,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:59,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:59,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137419327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:59,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:59,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:59,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:59,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:59,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137419347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:59,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:59,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:59,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137419347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:59,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:59,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:59,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:59,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:59,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:59,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137419348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:59,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:59,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:59,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:59,360 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:59,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T21:15:59,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:15:59,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:15:59,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:15:59,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:59,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:59,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:15:59,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T21:15:59,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/766bc78b98f0445a88088b04aff78c83 2024-11-20T21:15:59,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/49e3e8bfb60f41d3a8f4b7a7a5cd2784 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/49e3e8bfb60f41d3a8f4b7a7a5cd2784 2024-11-20T21:15:59,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/49e3e8bfb60f41d3a8f4b7a7a5cd2784, entries=150, sequenceid=107, filesize=11.7 K 2024-11-20T21:15:59,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/67f6b210352e406598d7f3180f7003f3 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/67f6b210352e406598d7f3180f7003f3 2024-11-20T21:15:59,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/67f6b210352e406598d7f3180f7003f3, entries=150, sequenceid=107, filesize=11.7 K 2024-11-20T21:15:59,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/766bc78b98f0445a88088b04aff78c83 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/766bc78b98f0445a88088b04aff78c83 2024-11-20T21:15:59,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/766bc78b98f0445a88088b04aff78c83, entries=150, sequenceid=107, filesize=11.7 K 2024-11-20T21:15:59,434 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 6c66f98c982744caf390af289e0ec6cf in 904ms, sequenceid=107, compaction requested=true 2024-11-20T21:15:59,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:15:59,435 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:59,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c66f98c982744caf390af289e0ec6cf:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:15:59,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:59,435 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:59,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c66f98c982744caf390af289e0ec6cf:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:15:59,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:59,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c66f98c982744caf390af289e0ec6cf:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:15:59,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:59,445 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:59,445 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 6c66f98c982744caf390af289e0ec6cf/A is initiating minor compaction (all files) 2024-11-20T21:15:59,445 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c66f98c982744caf390af289e0ec6cf/A in TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:15:59,445 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/fe4c3aafc97c4cd7b69f629d26da4a23, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/35e6431dc25b4a15adccc62a7c715ac4, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/49e3e8bfb60f41d3a8f4b7a7a5cd2784] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp, totalSize=37.5 K 2024-11-20T21:15:59,447 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:59,447 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 6c66f98c982744caf390af289e0ec6cf/B is initiating minor compaction (all files) 2024-11-20T21:15:59,447 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c66f98c982744caf390af289e0ec6cf/B in TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:15:59,447 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/9893e32ac671412b9e433d69c35f921b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/713a1ce5d04a410baf849d281f638277, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/67f6b210352e406598d7f3180f7003f3] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp, totalSize=35.3 K 2024-11-20T21:15:59,447 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe4c3aafc97c4cd7b69f629d26da4a23, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732137356217 2024-11-20T21:15:59,448 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 9893e32ac671412b9e433d69c35f921b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732137356217 2024-11-20T21:15:59,448 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35e6431dc25b4a15adccc62a7c715ac4, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732137357481 2024-11-20T21:15:59,451 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 713a1ce5d04a410baf849d281f638277, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732137357481 2024-11-20T21:15:59,451 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49e3e8bfb60f41d3a8f4b7a7a5cd2784, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732137357874 2024-11-20T21:15:59,452 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 67f6b210352e406598d7f3180f7003f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732137357874 2024-11-20T21:15:59,467 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c66f98c982744caf390af289e0ec6cf#A#compaction#431 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:59,467 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/04ee58eb8fee40239a9eff6a01f199d7 is 50, key is test_row_0/A:col10/1732137357874/Put/seqid=0 2024-11-20T21:15:59,471 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c66f98c982744caf390af289e0ec6cf#B#compaction#432 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:59,471 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/507e10e21e6e4a14bd39e918105c72fa is 50, key is test_row_0/B:col10/1732137357874/Put/seqid=0 2024-11-20T21:15:59,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742340_1516 (size=12104) 2024-11-20T21:15:59,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742341_1517 (size=12104) 2024-11-20T21:15:59,508 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/507e10e21e6e4a14bd39e918105c72fa as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/507e10e21e6e4a14bd39e918105c72fa 2024-11-20T21:15:59,512 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c66f98c982744caf390af289e0ec6cf/B of 6c66f98c982744caf390af289e0ec6cf into 507e10e21e6e4a14bd39e918105c72fa(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:59,512 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:15:59,512 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., storeName=6c66f98c982744caf390af289e0ec6cf/B, priority=13, startTime=1732137359435; duration=0sec 2024-11-20T21:15:59,512 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:15:59,512 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c66f98c982744caf390af289e0ec6cf:B 2024-11-20T21:15:59,512 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:15:59,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-20T21:15:59,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:15:59,514 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 6c66f98c982744caf390af289e0ec6cf 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-11-20T21:15:59,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=A 2024-11-20T21:15:59,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:59,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=B 2024-11-20T21:15:59,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:59,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=C 2024-11-20T21:15:59,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:15:59,516 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:15:59,516 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 6c66f98c982744caf390af289e0ec6cf/C is initiating minor compaction (all files) 2024-11-20T21:15:59,516 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c66f98c982744caf390af289e0ec6cf/C in TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:15:59,516 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/10875b29b3ef4838b4ac181225e27b62, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/afc83d782713497cbd9dbd2c48acf402, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/766bc78b98f0445a88088b04aff78c83] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp, totalSize=35.3 K 2024-11-20T21:15:59,516 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 10875b29b3ef4838b4ac181225e27b62, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1732137356217 2024-11-20T21:15:59,517 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting afc83d782713497cbd9dbd2c48acf402, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1732137357481 2024-11-20T21:15:59,517 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 766bc78b98f0445a88088b04aff78c83, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732137357874 2024-11-20T21:15:59,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/75b4fd02a2514c92a3244c79079f90b2 is 50, key is test_row_0/A:col10/1732137358674/Put/seqid=0 2024-11-20T21:15:59,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742342_1518 (size=12151) 2024-11-20T21:15:59,543 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c66f98c982744caf390af289e0ec6cf#C#compaction#434 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:15:59,543 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/9f77bfa8c6194db1acd19de9765df72e is 50, key is test_row_0/C:col10/1732137357874/Put/seqid=0 2024-11-20T21:15:59,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742343_1519 (size=12104) 2024-11-20T21:15:59,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T21:15:59,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:15:59,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:15:59,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:59,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137419816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:59,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:59,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:59,839 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:59,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:59,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137419847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:59,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:59,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:59,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:59,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:59,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137419867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:59,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:59,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137419868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:59,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:59,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:59,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:59,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:59,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:59,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:59,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:59,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137419870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:59,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:59,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:59,885 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:59,902 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/04ee58eb8fee40239a9eff6a01f199d7 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/04ee58eb8fee40239a9eff6a01f199d7 2024-11-20T21:15:59,909 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c66f98c982744caf390af289e0ec6cf/A of 6c66f98c982744caf390af289e0ec6cf into 04ee58eb8fee40239a9eff6a01f199d7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:59,910 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:15:59,910 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., storeName=6c66f98c982744caf390af289e0ec6cf/A, priority=13, startTime=1732137359434; duration=0sec 2024-11-20T21:15:59,910 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:59,910 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c66f98c982744caf390af289e0ec6cf:A 2024-11-20T21:15:59,934 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/75b4fd02a2514c92a3244c79079f90b2 2024-11-20T21:15:59,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/9dbd9f120f89495aaa144c3d473a66ae is 50, key is test_row_0/B:col10/1732137358674/Put/seqid=0 2024-11-20T21:15:59,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742344_1520 (size=12151) 2024-11-20T21:15:59,955 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/9dbd9f120f89495aaa144c3d473a66ae 2024-11-20T21:15:59,956 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/9f77bfa8c6194db1acd19de9765df72e as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/9f77bfa8c6194db1acd19de9765df72e 2024-11-20T21:15:59,956 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:15:59,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137419948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:15:59,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:59,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:15:59,966 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:15:59,977 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c66f98c982744caf390af289e0ec6cf/C of 6c66f98c982744caf390af289e0ec6cf into 9f77bfa8c6194db1acd19de9765df72e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:15:59,977 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:15:59,977 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., storeName=6c66f98c982744caf390af289e0ec6cf/C, priority=13, startTime=1732137359436; duration=0sec 2024-11-20T21:15:59,977 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:15:59,977 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c66f98c982744caf390af289e0ec6cf:C 2024-11-20T21:15:59,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/29b7437681524c3db9cbc75b610f1f1a is 50, key is test_row_0/C:col10/1732137358674/Put/seqid=0 2024-11-20T21:15:59,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742345_1521 (size=12151) 2024-11-20T21:15:59,990 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/29b7437681524c3db9cbc75b610f1f1a 2024-11-20T21:15:59,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/75b4fd02a2514c92a3244c79079f90b2 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/75b4fd02a2514c92a3244c79079f90b2 2024-11-20T21:15:59,999 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/75b4fd02a2514c92a3244c79079f90b2, entries=150, sequenceid=144, filesize=11.9 K 2024-11-20T21:16:00,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/9dbd9f120f89495aaa144c3d473a66ae as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/9dbd9f120f89495aaa144c3d473a66ae 2024-11-20T21:16:00,007 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/9dbd9f120f89495aaa144c3d473a66ae, entries=150, sequenceid=144, filesize=11.9 K 2024-11-20T21:16:00,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/29b7437681524c3db9cbc75b610f1f1a as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/29b7437681524c3db9cbc75b610f1f1a 2024-11-20T21:16:00,010 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/29b7437681524c3db9cbc75b610f1f1a, entries=150, sequenceid=144, filesize=11.9 K 2024-11-20T21:16:00,011 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for 6c66f98c982744caf390af289e0ec6cf in 497ms, sequenceid=144, compaction requested=false 2024-11-20T21:16:00,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:00,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:00,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-20T21:16:00,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-20T21:16:00,018 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-20T21:16:00,018 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 964 msec 2024-11-20T21:16:00,023 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 973 msec 2024-11-20T21:16:00,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:16:00,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c66f98c982744caf390af289e0ec6cf 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T21:16:00,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=A 2024-11-20T21:16:00,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:00,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=B 2024-11-20T21:16:00,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:00,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=C 2024-11-20T21:16:00,182 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:00,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/1ccd6d31abf84bc4b43cc67994f4d200 is 50, key is test_row_0/A:col10/1732137360180/Put/seqid=0 2024-11-20T21:16:00,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-20T21:16:00,188 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:16:00,189 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:16:00,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-20T21:16:00,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T21:16:00,192 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:16:00,193 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:16:00,193 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:16:00,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742346_1522 (size=14541) 2024-11-20T21:16:00,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T21:16:00,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732137360265, max timestamp=1732137360319Num uniques 859; 2024-11-20T21:16:00,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T21:16:00,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732137360265, max timestamp=1732137360319Num uniques 859; 2024-11-20T21:16:00,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T21:16:00,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732137360265, max timestamp=1732137360319Num uniques 859; 2024-11-20T21:16:00,327 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T21:16:00,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-20T21:16:00,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:00,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:16:00,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:00,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:00,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:00,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:00,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-20T21:16:00,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:00,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:16:00,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:00,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:00,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:00,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:00,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T21:16:00,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:00,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137420569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:00,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:00,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:00,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:00,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/1ccd6d31abf84bc4b43cc67994f4d200 2024-11-20T21:16:00,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/3a0f43a3218f446ba3724862d0181b52 is 50, key is test_row_0/B:col10/1732137360180/Put/seqid=0 2024-11-20T21:16:00,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-20T21:16:00,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:00,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:16:00,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:00,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:00,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:00,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:00,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742347_1523 (size=12151) 2024-11-20T21:16:00,702 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/3a0f43a3218f446ba3724862d0181b52 2024-11-20T21:16:00,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:00,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137420687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:00,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:00,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:00,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:00,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/b1e791265fb4486380e2c8cf378966df is 50, key is test_row_0/C:col10/1732137360180/Put/seqid=0 2024-11-20T21:16:00,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742348_1524 (size=12151) 2024-11-20T21:16:00,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/b1e791265fb4486380e2c8cf378966df 2024-11-20T21:16:00,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/1ccd6d31abf84bc4b43cc67994f4d200 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/1ccd6d31abf84bc4b43cc67994f4d200 2024-11-20T21:16:00,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/1ccd6d31abf84bc4b43cc67994f4d200, entries=200, sequenceid=156, filesize=14.2 K 2024-11-20T21:16:00,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/3a0f43a3218f446ba3724862d0181b52 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/3a0f43a3218f446ba3724862d0181b52 2024-11-20T21:16:00,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/3a0f43a3218f446ba3724862d0181b52, entries=150, sequenceid=156, filesize=11.9 K 2024-11-20T21:16:00,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/b1e791265fb4486380e2c8cf378966df as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/b1e791265fb4486380e2c8cf378966df 2024-11-20T21:16:00,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/b1e791265fb4486380e2c8cf378966df, entries=150, sequenceid=156, filesize=11.9 K 2024-11-20T21:16:00,786 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 6c66f98c982744caf390af289e0ec6cf in 605ms, sequenceid=156, compaction requested=true 2024-11-20T21:16:00,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:00,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c66f98c982744caf390af289e0ec6cf:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:16:00,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:00,786 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:16:00,787 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:16:00,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c66f98c982744caf390af289e0ec6cf:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:16:00,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:00,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c66f98c982744caf390af289e0ec6cf:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:16:00,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:16:00,788 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38796 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:16:00,788 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 6c66f98c982744caf390af289e0ec6cf/A is initiating minor compaction (all files) 2024-11-20T21:16:00,788 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c66f98c982744caf390af289e0ec6cf/A in TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:00,788 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/04ee58eb8fee40239a9eff6a01f199d7, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/75b4fd02a2514c92a3244c79079f90b2, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/1ccd6d31abf84bc4b43cc67994f4d200] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp, totalSize=37.9 K 2024-11-20T21:16:00,789 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04ee58eb8fee40239a9eff6a01f199d7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732137357874 2024-11-20T21:16:00,789 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36406 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:16:00,789 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 6c66f98c982744caf390af289e0ec6cf/B is initiating minor compaction (all files) 2024-11-20T21:16:00,789 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c66f98c982744caf390af289e0ec6cf/B in TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:00,789 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/507e10e21e6e4a14bd39e918105c72fa, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/9dbd9f120f89495aaa144c3d473a66ae, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/3a0f43a3218f446ba3724862d0181b52] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp, totalSize=35.6 K 2024-11-20T21:16:00,789 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75b4fd02a2514c92a3244c79079f90b2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732137358649 2024-11-20T21:16:00,790 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ccd6d31abf84bc4b43cc67994f4d200, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732137359772 2024-11-20T21:16:00,790 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 507e10e21e6e4a14bd39e918105c72fa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732137357874 2024-11-20T21:16:00,790 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dbd9f120f89495aaa144c3d473a66ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732137358649 2024-11-20T21:16:00,791 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a0f43a3218f446ba3724862d0181b52, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732137359772 2024-11-20T21:16:00,800 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c66f98c982744caf390af289e0ec6cf#A#compaction#440 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:00,801 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c66f98c982744caf390af289e0ec6cf#B#compaction#441 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:00,801 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/c13f890c94c4418e9dfc39cb24cc4d3b is 50, key is test_row_0/A:col10/1732137360180/Put/seqid=0 2024-11-20T21:16:00,801 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/86ebd761223e41cfa48e6db5b280e65a is 50, key is test_row_0/B:col10/1732137360180/Put/seqid=0 2024-11-20T21:16:00,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T21:16:00,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-20T21:16:00,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:00,829 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 6c66f98c982744caf390af289e0ec6cf 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-20T21:16:00,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=A 2024-11-20T21:16:00,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:16:00,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=B 2024-11-20T21:16:00,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:16:00,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=C 2024-11-20T21:16:00,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:16:00,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742350_1526 (size=12357) 2024-11-20T21:16:00,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742349_1525 (size=12357) 2024-11-20T21:16:00,849 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/c13f890c94c4418e9dfc39cb24cc4d3b as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/c13f890c94c4418e9dfc39cb24cc4d3b 2024-11-20T21:16:00,856 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c66f98c982744caf390af289e0ec6cf/A of 6c66f98c982744caf390af289e0ec6cf into c13f890c94c4418e9dfc39cb24cc4d3b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:00,856 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:00,856 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., storeName=6c66f98c982744caf390af289e0ec6cf/A, priority=13, startTime=1732137360786; duration=0sec 2024-11-20T21:16:00,856 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:16:00,856 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c66f98c982744caf390af289e0ec6cf:A 2024-11-20T21:16:00,856 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:16:00,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/2d49867e21574bf5ab3e5ea026827a23 is 50, key is test_row_0/A:col10/1732137360560/Put/seqid=0 2024-11-20T21:16:00,860 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36406 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:16:00,860 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 6c66f98c982744caf390af289e0ec6cf/C is initiating minor compaction (all files) 2024-11-20T21:16:00,860 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c66f98c982744caf390af289e0ec6cf/C in TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:00,861 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/9f77bfa8c6194db1acd19de9765df72e, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/29b7437681524c3db9cbc75b610f1f1a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/b1e791265fb4486380e2c8cf378966df] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp, totalSize=35.6 K 2024-11-20T21:16:00,861 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f77bfa8c6194db1acd19de9765df72e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732137357874 2024-11-20T21:16:00,861 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29b7437681524c3db9cbc75b610f1f1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1732137358649 2024-11-20T21:16:00,862 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1e791265fb4486380e2c8cf378966df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732137359772 2024-11-20T21:16:00,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742351_1527 (size=12151) 2024-11-20T21:16:00,883 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c66f98c982744caf390af289e0ec6cf#C#compaction#443 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:00,883 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/52ae69b05cd54a3b9ace18b7739f17db is 50, key is test_row_0/C:col10/1732137360180/Put/seqid=0 2024-11-20T21:16:00,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:16:00,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:16:00,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742352_1528 (size=12357) 2024-11-20T21:16:00,900 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/52ae69b05cd54a3b9ace18b7739f17db as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/52ae69b05cd54a3b9ace18b7739f17db 2024-11-20T21:16:00,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:00,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137420909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:00,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:00,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:00,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:00,925 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c66f98c982744caf390af289e0ec6cf/C of 6c66f98c982744caf390af289e0ec6cf into 52ae69b05cd54a3b9ace18b7739f17db(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:00,925 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:00,925 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., storeName=6c66f98c982744caf390af289e0ec6cf/C, priority=13, startTime=1732137360787; duration=0sec 2024-11-20T21:16:00,925 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:00,925 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c66f98c982744caf390af289e0ec6cf:C 2024-11-20T21:16:00,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:00,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137420911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:00,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:00,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:00,932 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:00,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:00,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137420918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:00,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:00,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137420919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:00,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:00,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137420925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:00,938 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:00,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:00,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:00,939 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:01,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:01,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137421027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:01,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:01,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:01,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:01,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:01,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137421037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:01,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:01,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:01,046 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:01,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:01,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137421046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:01,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:01,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137421047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:01,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:01,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:01,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:01,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:01,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:01,060 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:01,251 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/86ebd761223e41cfa48e6db5b280e65a as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/86ebd761223e41cfa48e6db5b280e65a 2024-11-20T21:16:01,256 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c66f98c982744caf390af289e0ec6cf/B of 6c66f98c982744caf390af289e0ec6cf into 86ebd761223e41cfa48e6db5b280e65a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:01,256 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:01,256 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., storeName=6c66f98c982744caf390af289e0ec6cf/B, priority=13, startTime=1732137360786; duration=0sec 2024-11-20T21:16:01,256 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:01,256 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c66f98c982744caf390af289e0ec6cf:B 2024-11-20T21:16:01,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:01,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137421248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:01,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:01,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:01,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:01,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137421249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:01,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:01,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:01,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:01,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:01,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137421257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:01,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:01,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:01,270 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:01,280 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/2d49867e21574bf5ab3e5ea026827a23 2024-11-20T21:16:01,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:01,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137421266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:01,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:01,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137421267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:01,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:01,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:01,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:01,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:01,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:01,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:01,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/54ad62d07aa044bd90bbe1c675f114d3 is 50, key is test_row_0/B:col10/1732137360560/Put/seqid=0 2024-11-20T21:16:01,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742353_1529 (size=12151) 2024-11-20T21:16:01,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T21:16:01,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:01,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137421577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:01,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:01,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137421579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:01,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:01,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:01,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:01,597 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:01,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:01,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137421589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:01,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:01,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137421589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:01,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:01,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:01,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:01,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:01,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:01,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:01,709 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/54ad62d07aa044bd90bbe1c675f114d3 2024-11-20T21:16:01,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/18fad13f0fc949f8a5bdf0ccfc8cce73 is 50, key is test_row_0/C:col10/1732137360560/Put/seqid=0 2024-11-20T21:16:01,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742354_1530 (size=12151) 2024-11-20T21:16:01,754 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/18fad13f0fc949f8a5bdf0ccfc8cce73 2024-11-20T21:16:01,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/2d49867e21574bf5ab3e5ea026827a23 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/2d49867e21574bf5ab3e5ea026827a23 2024-11-20T21:16:01,776 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/2d49867e21574bf5ab3e5ea026827a23, entries=150, sequenceid=193, filesize=11.9 K 2024-11-20T21:16:01,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:01,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137421767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:01,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:01,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/54ad62d07aa044bd90bbe1c675f114d3 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/54ad62d07aa044bd90bbe1c675f114d3 2024-11-20T21:16:01,785 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/54ad62d07aa044bd90bbe1c675f114d3, entries=150, sequenceid=193, filesize=11.9 K 2024-11-20T21:16:01,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/18fad13f0fc949f8a5bdf0ccfc8cce73 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/18fad13f0fc949f8a5bdf0ccfc8cce73 2024-11-20T21:16:01,796 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/18fad13f0fc949f8a5bdf0ccfc8cce73, entries=150, sequenceid=193, filesize=11.9 K 2024-11-20T21:16:01,797 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 6c66f98c982744caf390af289e0ec6cf in 967ms, sequenceid=193, compaction requested=false 2024-11-20T21:16:01,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:01,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:01,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-20T21:16:01,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-20T21:16:01,800 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-20T21:16:01,800 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6050 sec 2024-11-20T21:16:01,805 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.6120 sec 2024-11-20T21:16:02,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:16:02,111 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c66f98c982744caf390af289e0ec6cf 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-20T21:16:02,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=A 2024-11-20T21:16:02,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:02,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=B 2024-11-20T21:16:02,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:02,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=C 2024-11-20T21:16:02,111 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:02,123 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/3b0a3ccb07c34a739a9c2917a5ef323b is 50, key is test_row_0/A:col10/1732137360912/Put/seqid=0 2024-11-20T21:16:02,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742355_1531 (size=14541) 2024-11-20T21:16:02,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732137362169, max timestamp=1732137362200Num uniques 859; 2024-11-20T21:16:02,203 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T21:16:02,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732137362169, max timestamp=1732137362200Num uniques 859; 2024-11-20T21:16:02,204 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T21:16:02,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732137362169, max timestamp=1732137362200Num uniques 859; 2024-11-20T21:16:02,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T21:16:02,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:02,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137422271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:02,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:02,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137422272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:02,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:02,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:02,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137422278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:02,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:02,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137422279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:02,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:02,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:02,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-20T21:16:02,338 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:16:02,338 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:16:02,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-20T21:16:02,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T21:16:02,356 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:16:02,360 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:16:02,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:16:02,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:02,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137422387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:02,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:02,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:02,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137422397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:02,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:02,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:02,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137422412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:02,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:02,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:02,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137422413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:02,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:02,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T21:16:02,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T21:16:02,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:02,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:16:02,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:02,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:02,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:02,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:02,549 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/3b0a3ccb07c34a739a9c2917a5ef323b 2024-11-20T21:16:02,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/9f7ec3b4a422400ab9bc2ea9e291864d is 50, key is test_row_0/B:col10/1732137360912/Put/seqid=0 2024-11-20T21:16:02,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742356_1532 (size=12151) 2024-11-20T21:16:02,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:02,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137422616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:02,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:02,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:02,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137422617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:02,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:02,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:02,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137422637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:02,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:02,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137422637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:02,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,644 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:02,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:02,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T21:16:02,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T21:16:02,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:02,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:16:02,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:02,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:02,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:02,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:02,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:02,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137422787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:02,803 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 0b16ab3f13b7,36987,1732137236536 for 'test_row_2' in 6c66f98c982744caf390af289e0ec6cf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T21:16:02,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:02,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T21:16:02,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:02,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:16:02,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:02,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:02,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:02,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:02,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:02,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137422941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:02,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:02,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137422941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:02,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:02,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:02,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/9f7ec3b4a422400ab9bc2ea9e291864d 2024-11-20T21:16:02,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:02,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137422961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:02,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:02,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137422965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:02,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,969 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:02,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:02,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:02,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T21:16:02,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T21:16:02,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:02,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:16:02,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:02,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:02,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:03,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:03,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/422dee926cfa45e0b12d3e03f3157e06 is 50, key is test_row_0/C:col10/1732137360912/Put/seqid=0 2024-11-20T21:16:03,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742357_1533 (size=12151) 2024-11-20T21:16:03,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/422dee926cfa45e0b12d3e03f3157e06 2024-11-20T21:16:03,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/3b0a3ccb07c34a739a9c2917a5ef323b as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/3b0a3ccb07c34a739a9c2917a5ef323b 2024-11-20T21:16:03,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/3b0a3ccb07c34a739a9c2917a5ef323b, entries=200, sequenceid=207, filesize=14.2 K 2024-11-20T21:16:03,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/9f7ec3b4a422400ab9bc2ea9e291864d as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/9f7ec3b4a422400ab9bc2ea9e291864d 2024-11-20T21:16:03,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/9f7ec3b4a422400ab9bc2ea9e291864d, entries=150, sequenceid=207, filesize=11.9 K 2024-11-20T21:16:03,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/422dee926cfa45e0b12d3e03f3157e06 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/422dee926cfa45e0b12d3e03f3157e06 2024-11-20T21:16:03,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/422dee926cfa45e0b12d3e03f3157e06, entries=150, sequenceid=207, filesize=11.9 K 2024-11-20T21:16:03,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 6c66f98c982744caf390af289e0ec6cf in 998ms, sequenceid=207, compaction requested=true 2024-11-20T21:16:03,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:03,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c66f98c982744caf390af289e0ec6cf:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:16:03,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:03,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c66f98c982744caf390af289e0ec6cf:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:16:03,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:16:03,109 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:16:03,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c66f98c982744caf390af289e0ec6cf:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:16:03,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T21:16:03,109 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:16:03,110 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:16:03,110 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 6c66f98c982744caf390af289e0ec6cf/A is initiating minor compaction (all files) 2024-11-20T21:16:03,110 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c66f98c982744caf390af289e0ec6cf/A in TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:03,110 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/c13f890c94c4418e9dfc39cb24cc4d3b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/2d49867e21574bf5ab3e5ea026827a23, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/3b0a3ccb07c34a739a9c2917a5ef323b] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp, totalSize=38.1 K 2024-11-20T21:16:03,110 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:16:03,110 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 6c66f98c982744caf390af289e0ec6cf/B is initiating minor compaction (all files) 2024-11-20T21:16:03,110 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c66f98c982744caf390af289e0ec6cf/B in TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:03,110 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/86ebd761223e41cfa48e6db5b280e65a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/54ad62d07aa044bd90bbe1c675f114d3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/9f7ec3b4a422400ab9bc2ea9e291864d] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp, totalSize=35.8 K 2024-11-20T21:16:03,111 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting c13f890c94c4418e9dfc39cb24cc4d3b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732137359772 2024-11-20T21:16:03,111 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 86ebd761223e41cfa48e6db5b280e65a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732137359772 2024-11-20T21:16:03,111 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d49867e21574bf5ab3e5ea026827a23, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732137360500 2024-11-20T21:16:03,111 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 54ad62d07aa044bd90bbe1c675f114d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732137360500 2024-11-20T21:16:03,111 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b0a3ccb07c34a739a9c2917a5ef323b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732137360907 2024-11-20T21:16:03,112 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f7ec3b4a422400ab9bc2ea9e291864d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732137360912 2024-11-20T21:16:03,129 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c66f98c982744caf390af289e0ec6cf#A#compaction#449 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:03,130 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/da432cb55aa04bf68dd01b01eb11f61a is 50, key is test_row_0/A:col10/1732137360912/Put/seqid=0 2024-11-20T21:16:03,139 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c66f98c982744caf390af289e0ec6cf#B#compaction#450 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:03,139 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/ec939cd02ac0451baea9af8d95aceccc is 50, key is test_row_0/B:col10/1732137360912/Put/seqid=0 2024-11-20T21:16:03,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-20T21:16:03,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:03,159 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 6c66f98c982744caf390af289e0ec6cf 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-20T21:16:03,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=A 2024-11-20T21:16:03,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:16:03,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=B 2024-11-20T21:16:03,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:16:03,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=C 2024-11-20T21:16:03,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:16:03,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/ab581d7384bf4c7a9a0187ccc0f815d5 is 50, key is test_row_0/A:col10/1732137362274/Put/seqid=0 2024-11-20T21:16:03,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742359_1535 (size=12459) 2024-11-20T21:16:03,210 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/ec939cd02ac0451baea9af8d95aceccc as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/ec939cd02ac0451baea9af8d95aceccc 2024-11-20T21:16:03,215 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c66f98c982744caf390af289e0ec6cf/B of 6c66f98c982744caf390af289e0ec6cf into ec939cd02ac0451baea9af8d95aceccc(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:03,215 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:03,215 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., storeName=6c66f98c982744caf390af289e0ec6cf/B, priority=13, startTime=1732137363109; duration=0sec 2024-11-20T21:16:03,215 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:16:03,215 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c66f98c982744caf390af289e0ec6cf:B 2024-11-20T21:16:03,215 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:16:03,219 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:16:03,219 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 6c66f98c982744caf390af289e0ec6cf/C is initiating minor compaction (all files) 2024-11-20T21:16:03,219 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c66f98c982744caf390af289e0ec6cf/C in TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:03,219 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/52ae69b05cd54a3b9ace18b7739f17db, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/18fad13f0fc949f8a5bdf0ccfc8cce73, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/422dee926cfa45e0b12d3e03f3157e06] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp, totalSize=35.8 K 2024-11-20T21:16:03,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742358_1534 (size=12459) 2024-11-20T21:16:03,223 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 52ae69b05cd54a3b9ace18b7739f17db, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732137359772 2024-11-20T21:16:03,223 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 18fad13f0fc949f8a5bdf0ccfc8cce73, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1732137360500 2024-11-20T21:16:03,224 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 422dee926cfa45e0b12d3e03f3157e06, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732137360912 2024-11-20T21:16:03,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742360_1536 (size=12151) 2024-11-20T21:16:03,237 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/ab581d7384bf4c7a9a0187ccc0f815d5 2024-11-20T21:16:03,252 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/da432cb55aa04bf68dd01b01eb11f61a as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/da432cb55aa04bf68dd01b01eb11f61a 2024-11-20T21:16:03,262 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c66f98c982744caf390af289e0ec6cf#C#compaction#452 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:03,262 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/6fc07c45cd3146d1a0f6011e08400353 is 50, key is test_row_0/C:col10/1732137360912/Put/seqid=0 2024-11-20T21:16:03,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/fd8cc43102db4aeba015a62d6c7d6f4c is 50, key is test_row_0/B:col10/1732137362274/Put/seqid=0 2024-11-20T21:16:03,281 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c66f98c982744caf390af289e0ec6cf/A of 6c66f98c982744caf390af289e0ec6cf into da432cb55aa04bf68dd01b01eb11f61a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:03,281 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:03,281 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., storeName=6c66f98c982744caf390af289e0ec6cf/A, priority=13, startTime=1732137363109; duration=0sec 2024-11-20T21:16:03,281 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:03,282 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c66f98c982744caf390af289e0ec6cf:A 2024-11-20T21:16:03,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742362_1538 (size=12459) 2024-11-20T21:16:03,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742361_1537 (size=12151) 2024-11-20T21:16:03,339 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/fd8cc43102db4aeba015a62d6c7d6f4c 2024-11-20T21:16:03,343 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/6fc07c45cd3146d1a0f6011e08400353 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/6fc07c45cd3146d1a0f6011e08400353 2024-11-20T21:16:03,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/8d24c58d5ada4691a329761e58dd63c5 is 50, key is test_row_0/C:col10/1732137362274/Put/seqid=0 2024-11-20T21:16:03,361 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c66f98c982744caf390af289e0ec6cf/C of 6c66f98c982744caf390af289e0ec6cf into 6fc07c45cd3146d1a0f6011e08400353(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:03,361 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:03,361 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., storeName=6c66f98c982744caf390af289e0ec6cf/C, priority=13, startTime=1732137363109; duration=0sec 2024-11-20T21:16:03,361 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:03,361 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c66f98c982744caf390af289e0ec6cf:C 2024-11-20T21:16:03,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742363_1539 (size=12151) 2024-11-20T21:16:03,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:16:03,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:16:03,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:03,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137423477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:03,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:03,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:03,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:03,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T21:16:03,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:03,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137423488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:03,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:03,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:03,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:03,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:03,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137423492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:03,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:03,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:03,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:03,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:03,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137423496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:03,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:03,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:03,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:03,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:03,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137423608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:03,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:03,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:03,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:03,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:03,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137423608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:03,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:03,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:03,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:03,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:03,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137423616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:03,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:03,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:03,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:03,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:03,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137423627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:03,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:03,772 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/8d24c58d5ada4691a329761e58dd63c5 2024-11-20T21:16:03,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/ab581d7384bf4c7a9a0187ccc0f815d5 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/ab581d7384bf4c7a9a0187ccc0f815d5 2024-11-20T21:16:03,782 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/ab581d7384bf4c7a9a0187ccc0f815d5, entries=150, sequenceid=242, filesize=11.9 K 2024-11-20T21:16:03,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/fd8cc43102db4aeba015a62d6c7d6f4c as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/fd8cc43102db4aeba015a62d6c7d6f4c 2024-11-20T21:16:03,787 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/fd8cc43102db4aeba015a62d6c7d6f4c, entries=150, sequenceid=242, filesize=11.9 K 2024-11-20T21:16:03,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/8d24c58d5ada4691a329761e58dd63c5 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/8d24c58d5ada4691a329761e58dd63c5 2024-11-20T21:16:03,805 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/8d24c58d5ada4691a329761e58dd63c5, entries=150, sequenceid=242, filesize=11.9 K 2024-11-20T21:16:03,806 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 6c66f98c982744caf390af289e0ec6cf in 646ms, sequenceid=242, compaction requested=false 2024-11-20T21:16:03,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:03,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:03,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-20T21:16:03,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-20T21:16:03,809 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-20T21:16:03,809 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4470 sec 2024-11-20T21:16:03,811 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.4720 sec 2024-11-20T21:16:03,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:16:03,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c66f98c982744caf390af289e0ec6cf 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T21:16:03,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=A 2024-11-20T21:16:03,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:03,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=B 2024-11-20T21:16:03,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:03,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=C 2024-11-20T21:16:03,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:03,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/8e02a7ac44f147c8b19cc8e90fb21a75 is 50, key is test_row_0/A:col10/1732137363491/Put/seqid=0 2024-11-20T21:16:03,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742364_1540 (size=14591) 2024-11-20T21:16:03,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/8e02a7ac44f147c8b19cc8e90fb21a75 2024-11-20T21:16:03,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/f862725547d64ddd8df2ea758ef0c829 is 50, key is test_row_0/B:col10/1732137363491/Put/seqid=0 2024-11-20T21:16:03,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:03,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137423898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:03,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:03,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:03,924 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:03,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:03,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137423906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:03,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:03,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137423911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:03,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:03,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:03,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:03,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:03,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:03,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:03,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:03,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137423930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:03,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:03,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:03,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:03,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742365_1541 (size=12201) 2024-11-20T21:16:04,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:04,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137424027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:04,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:04,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:04,038 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:04,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:04,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137424037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:04,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:04,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:04,055 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:04,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:04,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137424039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:04,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:04,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:04,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:04,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:04,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137424047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:04,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:04,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:04,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:04,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:04,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137424247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:04,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:04,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:04,257 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:04,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:04,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137424256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:04,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:04,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:04,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:04,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:04,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137424270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:04,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:04,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:04,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:04,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:04,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137424279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:04,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:04,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:04,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:04,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/f862725547d64ddd8df2ea758ef0c829 2024-11-20T21:16:04,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/813688808fc548feaacf66a2dae86f8e is 50, key is test_row_0/C:col10/1732137363491/Put/seqid=0 2024-11-20T21:16:04,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742366_1542 (size=12201) 2024-11-20T21:16:04,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-20T21:16:04,519 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:16:04,521 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:16:04,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-20T21:16:04,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T21:16:04,537 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:16:04,538 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:16:04,538 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:16:04,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:04,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137424569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:04,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:04,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:04,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:04,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:04,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137424588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:04,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:04,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137424589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:04,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:04,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:04,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137424597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:04,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:04,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:04,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:04,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T21:16:04,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-20T21:16:04,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:04,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:16:04,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:04,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:04,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:04,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:04,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/813688808fc548feaacf66a2dae86f8e 2024-11-20T21:16:04,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/8e02a7ac44f147c8b19cc8e90fb21a75 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/8e02a7ac44f147c8b19cc8e90fb21a75 2024-11-20T21:16:04,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/8e02a7ac44f147c8b19cc8e90fb21a75, entries=200, sequenceid=257, filesize=14.2 K 2024-11-20T21:16:04,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/f862725547d64ddd8df2ea758ef0c829 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/f862725547d64ddd8df2ea758ef0c829 2024-11-20T21:16:04,797 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/f862725547d64ddd8df2ea758ef0c829, entries=150, sequenceid=257, filesize=11.9 K 2024-11-20T21:16:04,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/813688808fc548feaacf66a2dae86f8e as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/813688808fc548feaacf66a2dae86f8e 2024-11-20T21:16:04,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/813688808fc548feaacf66a2dae86f8e, entries=150, sequenceid=257, filesize=11.9 K 2024-11-20T21:16:04,801 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 6c66f98c982744caf390af289e0ec6cf in 968ms, sequenceid=257, compaction requested=true 2024-11-20T21:16:04,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:04,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c66f98c982744caf390af289e0ec6cf:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:16:04,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:04,801 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:16:04,801 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:16:04,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c66f98c982744caf390af289e0ec6cf:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:16:04,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:04,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c66f98c982744caf390af289e0ec6cf:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:16:04,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:16:04,803 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39201 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:16:04,803 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 6c66f98c982744caf390af289e0ec6cf/A is initiating minor compaction (all files) 2024-11-20T21:16:04,803 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c66f98c982744caf390af289e0ec6cf/A in TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:04,803 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/da432cb55aa04bf68dd01b01eb11f61a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/ab581d7384bf4c7a9a0187ccc0f815d5, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/8e02a7ac44f147c8b19cc8e90fb21a75] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp, totalSize=38.3 K 2024-11-20T21:16:04,803 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36811 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:16:04,803 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 6c66f98c982744caf390af289e0ec6cf/B is initiating minor compaction (all files) 2024-11-20T21:16:04,803 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c66f98c982744caf390af289e0ec6cf/B in TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:04,803 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/ec939cd02ac0451baea9af8d95aceccc, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/fd8cc43102db4aeba015a62d6c7d6f4c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/f862725547d64ddd8df2ea758ef0c829] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp, totalSize=35.9 K 2024-11-20T21:16:04,803 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting ec939cd02ac0451baea9af8d95aceccc, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732137360912 2024-11-20T21:16:04,804 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting da432cb55aa04bf68dd01b01eb11f61a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732137360912 2024-11-20T21:16:04,805 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting fd8cc43102db4aeba015a62d6c7d6f4c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732137362225 2024-11-20T21:16:04,805 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab581d7384bf4c7a9a0187ccc0f815d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732137362225 2024-11-20T21:16:04,805 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting f862725547d64ddd8df2ea758ef0c829, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732137363475 2024-11-20T21:16:04,805 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e02a7ac44f147c8b19cc8e90fb21a75, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732137363475 2024-11-20T21:16:04,815 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c66f98c982744caf390af289e0ec6cf#A#compaction#458 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:04,815 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/ddac5a8d79bf40f78940d78ddb2e8524 is 50, key is test_row_0/A:col10/1732137363491/Put/seqid=0 2024-11-20T21:16:04,818 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c66f98c982744caf390af289e0ec6cf#B#compaction#459 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:04,818 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/64c20dee55894fdfa83be0841aa97b09 is 50, key is test_row_0/B:col10/1732137363491/Put/seqid=0 2024-11-20T21:16:04,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732137363864, max timestamp=1732137364829Num uniques 859; 2024-11-20T21:16:04,833 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T21:16:04,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732137363864, max timestamp=1732137364829Num uniques 859; 2024-11-20T21:16:04,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T21:16:04,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732137363864, max timestamp=1732137364829Num uniques 859; 2024-11-20T21:16:04,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T21:16:04,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-20T21:16:04,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:04,846 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 6c66f98c982744caf390af289e0ec6cf 3/3 column families, dataSize=26.84 KB heapSize=66.23 KB 2024-11-20T21:16:04,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T21:16:04,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=A 2024-11-20T21:16:04,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:16:04,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=B 2024-11-20T21:16:04,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:16:04,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=C 2024-11-20T21:16:04,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:16:04,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742367_1543 (size=12611) 2024-11-20T21:16:04,858 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/64c20dee55894fdfa83be0841aa97b09 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/64c20dee55894fdfa83be0841aa97b09 2024-11-20T21:16:04,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/e4af9cab5d1a460aa7b4d3d9d8182df3 is 50, key is test_row_0/A:col10/1732137364843/Put/seqid=0 2024-11-20T21:16:04,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742368_1544 (size=12611) 2024-11-20T21:16:04,865 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c66f98c982744caf390af289e0ec6cf/B of 6c66f98c982744caf390af289e0ec6cf into 64c20dee55894fdfa83be0841aa97b09(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:04,865 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:04,865 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., storeName=6c66f98c982744caf390af289e0ec6cf/B, priority=13, startTime=1732137364801; duration=0sec 2024-11-20T21:16:04,865 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:16:04,865 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c66f98c982744caf390af289e0ec6cf:B 2024-11-20T21:16:04,865 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:16:04,866 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36811 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:16:04,866 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 6c66f98c982744caf390af289e0ec6cf/C is initiating minor compaction (all files) 2024-11-20T21:16:04,866 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c66f98c982744caf390af289e0ec6cf/C in TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:04,866 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/6fc07c45cd3146d1a0f6011e08400353, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/8d24c58d5ada4691a329761e58dd63c5, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/813688808fc548feaacf66a2dae86f8e] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp, totalSize=35.9 K 2024-11-20T21:16:04,867 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fc07c45cd3146d1a0f6011e08400353, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732137360912 2024-11-20T21:16:04,867 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d24c58d5ada4691a329761e58dd63c5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1732137362225 2024-11-20T21:16:04,867 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 813688808fc548feaacf66a2dae86f8e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732137363475 2024-11-20T21:16:04,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742369_1545 (size=14741) 2024-11-20T21:16:04,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:16:04,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:16:04,896 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c66f98c982744caf390af289e0ec6cf#C#compaction#461 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:04,897 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/ef0fdfe7e9134506baf0e33430b668f4 is 50, key is test_row_0/C:col10/1732137363491/Put/seqid=0 2024-11-20T21:16:04,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742370_1546 (size=12611) 2024-11-20T21:16:05,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732137365088, max timestamp=1732137365145Num uniques 859; 2024-11-20T21:16:05,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T21:16:05,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732137365088, max timestamp=1732137365145Num uniques 859; 2024-11-20T21:16:05,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T21:16:05,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732137365088, max timestamp=1732137365145Num uniques 859; 2024-11-20T21:16:05,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T21:16:05,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T21:16:05,266 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/ddac5a8d79bf40f78940d78ddb2e8524 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/ddac5a8d79bf40f78940d78ddb2e8524 2024-11-20T21:16:05,271 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c66f98c982744caf390af289e0ec6cf/A of 6c66f98c982744caf390af289e0ec6cf into ddac5a8d79bf40f78940d78ddb2e8524(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:05,271 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:05,271 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., storeName=6c66f98c982744caf390af289e0ec6cf/A, priority=13, startTime=1732137364801; duration=0sec 2024-11-20T21:16:05,271 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:05,271 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c66f98c982744caf390af289e0ec6cf:A 2024-11-20T21:16:05,289 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/e4af9cab5d1a460aa7b4d3d9d8182df3 2024-11-20T21:16:05,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732137365228, max timestamp=1732137365290Num uniques 1068; 2024-11-20T21:16:05,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-20T21:16:05,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732137365228, max timestamp=1732137365290Num uniques 1068; 2024-11-20T21:16:05,304 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-20T21:16:05,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1732137365228, max timestamp=1732137365290Num uniques 1068; 2024-11-20T21:16:05,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-20T21:16:05,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/dbcf395525054cd391668992c9e61acc is 50, key is test_row_0/B:col10/1732137364843/Put/seqid=0 2024-11-20T21:16:05,352 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/ef0fdfe7e9134506baf0e33430b668f4 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/ef0fdfe7e9134506baf0e33430b668f4 2024-11-20T21:16:05,359 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c66f98c982744caf390af289e0ec6cf/C of 6c66f98c982744caf390af289e0ec6cf into ef0fdfe7e9134506baf0e33430b668f4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:05,359 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:05,359 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., storeName=6c66f98c982744caf390af289e0ec6cf/C, priority=13, startTime=1732137364802; duration=0sec 2024-11-20T21:16:05,359 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:05,359 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c66f98c982744caf390af289e0ec6cf:C 2024-11-20T21:16:05,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742371_1547 (size=12301) 2024-11-20T21:16:05,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:05,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137425414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:05,438 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:05,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:05,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137425416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:05,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:05,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:05,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:05,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:05,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137425418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:05,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:05,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:05,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:05,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137425420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:05,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:05,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:05,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:05,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:05,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:05,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137425437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:05,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:05,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:05,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:05,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:05,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137425547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:05,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:05,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137425547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:05,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:05,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137425548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:05,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:05,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:05,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137425548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:05,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:05,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:05,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:05,566 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:05,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:05,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137425567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:05,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:05,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:05,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:05,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T21:16:05,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:05,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137425767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:05,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:05,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:05,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:05,791 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/dbcf395525054cd391668992c9e61acc 2024-11-20T21:16:05,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:05,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137425767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:05,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:05,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:05,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137425777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:05,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:05,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:05,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:05,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137425777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:05,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:05,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:05,793 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:05,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:05,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:05,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:05,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/92c13df3cdf94c61ba3d557cffdd087e is 50, key is test_row_0/C:col10/1732137364843/Put/seqid=0 2024-11-20T21:16:05,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:05,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137425787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:05,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:05,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:05,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:05,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742372_1548 (size=12301) 2024-11-20T21:16:06,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:06,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137426097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:06,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:06,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:06,132 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:06,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137426097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:06,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137426097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:06,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:06,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:06,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:06,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:06,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137426097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:06,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:06,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:06,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137426119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:06,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:06,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:06,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:06,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:06,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:06,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:06,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:06,245 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/92c13df3cdf94c61ba3d557cffdd087e 2024-11-20T21:16:06,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/e4af9cab5d1a460aa7b4d3d9d8182df3 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/e4af9cab5d1a460aa7b4d3d9d8182df3 2024-11-20T21:16:06,258 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/e4af9cab5d1a460aa7b4d3d9d8182df3, entries=200, sequenceid=279, filesize=14.4 K 2024-11-20T21:16:06,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/dbcf395525054cd391668992c9e61acc as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/dbcf395525054cd391668992c9e61acc 2024-11-20T21:16:06,265 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/dbcf395525054cd391668992c9e61acc, entries=150, sequenceid=279, filesize=12.0 K 2024-11-20T21:16:06,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/92c13df3cdf94c61ba3d557cffdd087e as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/92c13df3cdf94c61ba3d557cffdd087e 2024-11-20T21:16:06,272 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/92c13df3cdf94c61ba3d557cffdd087e, entries=150, sequenceid=279, filesize=12.0 K 2024-11-20T21:16:06,273 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=154.31 KB/158010 for 6c66f98c982744caf390af289e0ec6cf in 1427ms, sequenceid=279, compaction requested=false 2024-11-20T21:16:06,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:06,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:06,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-20T21:16:06,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-20T21:16:06,277 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-20T21:16:06,277 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7370 sec 2024-11-20T21:16:06,279 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.7560 sec 2024-11-20T21:16:06,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:16:06,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c66f98c982744caf390af289e0ec6cf 3/3 column families, dataSize=161.02 KB heapSize=450.47 KB 2024-11-20T21:16:06,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=A 2024-11-20T21:16:06,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T21:16:06,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=B 2024-11-20T21:16:06,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T21:16:06,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=C 2024-11-20T21:16:06,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T21:16:06,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/ff47b0d8b124421eb7e0db3e397cb480 is 50, key is test_row_0/A:col10/1732137365435/Put/seqid=0 2024-11-20T21:16:06,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742373_1549 (size=14741) 2024-11-20T21:16:06,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:06,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137426657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:06,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:06,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:06,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:06,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:06,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137426659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:06,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:06,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:06,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:06,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137426660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:06,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:06,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:06,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:06,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:06,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137426661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:06,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:06,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:06,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:06,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-20T21:16:06,677 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:16:06,677 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:16:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-20T21:16:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-20T21:16:06,678 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:16:06,679 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:16:06,679 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:16:06,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:06,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137426668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:06,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:06,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:06,690 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:06,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-20T21:16:06,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:06,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137426777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:06,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:06,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137426777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:06,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:06,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:06,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137426780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:06,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:06,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:06,790 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:06,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:06,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137426797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:06,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-20T21:16:06,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:06,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:16:06,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:06,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:06,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:06,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:06,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-20T21:16:06,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:06,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:16:06,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:06,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:06,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:06,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:06,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-20T21:16:07,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137426996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:07,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137426997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:07,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137426997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:07,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:07,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:07,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:07,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:07,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:07,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:07,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:07,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:07,004 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:07,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:07,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137427017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:07,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:07,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:07,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:07,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/ff47b0d8b124421eb7e0db3e397cb480 2024-11-20T21:16:07,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/141629e1a111430bb6a7d492aae9f0c6 is 50, key is test_row_0/B:col10/1732137365435/Put/seqid=0 2024-11-20T21:16:07,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742374_1550 (size=12301) 2024-11-20T21:16:07,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-20T21:16:07,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:07,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:16:07,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:07,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:07,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:07,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:07,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-20T21:16:07,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:07,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:16:07,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:07,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:07,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:07,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:07,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-20T21:16:07,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:07,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137427307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:07,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:07,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137427307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:07,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:07,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:07,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:07,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:07,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:07,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:07,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:07,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137427316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:07,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:07,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:07,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:07,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137427337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:07,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:07,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:07,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:07,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-20T21:16:07,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:07,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:16:07,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:07,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:07,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:07,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:07,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/141629e1a111430bb6a7d492aae9f0c6 2024-11-20T21:16:07,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/8879cdd7bbab4494b417dd0e5b5336e7 is 50, key is test_row_0/C:col10/1732137365435/Put/seqid=0 2024-11-20T21:16:07,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742375_1551 (size=12301) 2024-11-20T21:16:07,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/8879cdd7bbab4494b417dd0e5b5336e7 2024-11-20T21:16:07,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/ff47b0d8b124421eb7e0db3e397cb480 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/ff47b0d8b124421eb7e0db3e397cb480 2024-11-20T21:16:07,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/ff47b0d8b124421eb7e0db3e397cb480, entries=200, sequenceid=337, filesize=14.4 K 2024-11-20T21:16:07,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/141629e1a111430bb6a7d492aae9f0c6 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/141629e1a111430bb6a7d492aae9f0c6 2024-11-20T21:16:07,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/141629e1a111430bb6a7d492aae9f0c6, entries=150, sequenceid=337, filesize=12.0 K 2024-11-20T21:16:07,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/8879cdd7bbab4494b417dd0e5b5336e7 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/8879cdd7bbab4494b417dd0e5b5336e7 2024-11-20T21:16:07,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/8879cdd7bbab4494b417dd0e5b5336e7, entries=150, sequenceid=337, filesize=12.0 K 2024-11-20T21:16:07,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~450.42 KB/461232, currentSize=26.84 KB/27480 for 6c66f98c982744caf390af289e0ec6cf in 900ms, sequenceid=337, compaction requested=true 2024-11-20T21:16:07,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:07,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c66f98c982744caf390af289e0ec6cf:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:16:07,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:16:07,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c66f98c982744caf390af289e0ec6cf:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:16:07,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-20T21:16:07,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c66f98c982744caf390af289e0ec6cf:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:16:07,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-20T21:16:07,547 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:16:07,547 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:16:07,549 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:16:07,549 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37213 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:16:07,549 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 6c66f98c982744caf390af289e0ec6cf/C is initiating minor compaction (all files) 2024-11-20T21:16:07,549 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 6c66f98c982744caf390af289e0ec6cf/A is initiating minor compaction (all files) 2024-11-20T21:16:07,550 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c66f98c982744caf390af289e0ec6cf/C in TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:07,550 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c66f98c982744caf390af289e0ec6cf/A in TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:07,550 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/ef0fdfe7e9134506baf0e33430b668f4, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/92c13df3cdf94c61ba3d557cffdd087e, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/8879cdd7bbab4494b417dd0e5b5336e7] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp, totalSize=36.3 K 2024-11-20T21:16:07,550 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/ddac5a8d79bf40f78940d78ddb2e8524, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/e4af9cab5d1a460aa7b4d3d9d8182df3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/ff47b0d8b124421eb7e0db3e397cb480] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp, totalSize=41.1 K 2024-11-20T21:16:07,550 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddac5a8d79bf40f78940d78ddb2e8524, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732137363475 2024-11-20T21:16:07,550 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting ef0fdfe7e9134506baf0e33430b668f4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732137363475 2024-11-20T21:16:07,550 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 92c13df3cdf94c61ba3d557cffdd087e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732137363909 2024-11-20T21:16:07,550 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4af9cab5d1a460aa7b4d3d9d8182df3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732137363909 2024-11-20T21:16:07,551 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff47b0d8b124421eb7e0db3e397cb480, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1732137365414 2024-11-20T21:16:07,551 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8879cdd7bbab4494b417dd0e5b5336e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1732137365418 2024-11-20T21:16:07,559 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c66f98c982744caf390af289e0ec6cf#C#compaction#467 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:07,559 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/8c8c14ea72b64fe99c8edb13f8bb3c23 is 50, key is test_row_0/C:col10/1732137365435/Put/seqid=0 2024-11-20T21:16:07,563 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c66f98c982744caf390af289e0ec6cf#A#compaction#468 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:07,563 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/9f30037885f344038082f8a018844bd3 is 50, key is test_row_0/A:col10/1732137365435/Put/seqid=0 2024-11-20T21:16:07,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742376_1552 (size=12813) 2024-11-20T21:16:07,573 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/8c8c14ea72b64fe99c8edb13f8bb3c23 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/8c8c14ea72b64fe99c8edb13f8bb3c23 2024-11-20T21:16:07,578 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c66f98c982744caf390af289e0ec6cf/C of 6c66f98c982744caf390af289e0ec6cf into 8c8c14ea72b64fe99c8edb13f8bb3c23(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:07,578 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:07,578 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., storeName=6c66f98c982744caf390af289e0ec6cf/C, priority=13, startTime=1732137367545; duration=0sec 2024-11-20T21:16:07,578 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:16:07,578 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c66f98c982744caf390af289e0ec6cf:C 2024-11-20T21:16:07,578 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:16:07,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742377_1553 (size=12813) 2024-11-20T21:16:07,579 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37213 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:16:07,579 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 6c66f98c982744caf390af289e0ec6cf/B is initiating minor compaction (all files) 2024-11-20T21:16:07,579 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c66f98c982744caf390af289e0ec6cf/B in TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:07,579 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/64c20dee55894fdfa83be0841aa97b09, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/dbcf395525054cd391668992c9e61acc, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/141629e1a111430bb6a7d492aae9f0c6] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp, totalSize=36.3 K 2024-11-20T21:16:07,579 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 64c20dee55894fdfa83be0841aa97b09, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1732137363475 2024-11-20T21:16:07,580 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting dbcf395525054cd391668992c9e61acc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1732137363909 2024-11-20T21:16:07,580 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 141629e1a111430bb6a7d492aae9f0c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1732137365418 2024-11-20T21:16:07,585 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c66f98c982744caf390af289e0ec6cf#B#compaction#469 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:07,586 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/d307f9bd497b450da53ce0baec25fd56 is 50, key is test_row_0/B:col10/1732137365435/Put/seqid=0 2024-11-20T21:16:07,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742378_1554 (size=12813) 2024-11-20T21:16:07,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-20T21:16:07,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:07,611 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 6c66f98c982744caf390af289e0ec6cf 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-20T21:16:07,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=A 2024-11-20T21:16:07,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:07,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=B 2024-11-20T21:16:07,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:07,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=C 2024-11-20T21:16:07,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:07,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/c6bbddd1e5a345baa49cf35fff9d8cc2 is 50, key is test_row_0/A:col10/1732137366666/Put/seqid=0 2024-11-20T21:16:07,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742379_1555 (size=12301) 2024-11-20T21:16:07,620 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/c6bbddd1e5a345baa49cf35fff9d8cc2 2024-11-20T21:16:07,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/d927d40db64d4c2ebc36c5425329d7f4 is 50, key is test_row_0/B:col10/1732137366666/Put/seqid=0 2024-11-20T21:16:07,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742380_1556 (size=12301) 2024-11-20T21:16:07,630 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/d927d40db64d4c2ebc36c5425329d7f4 2024-11-20T21:16:07,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/65d48dedf6794638bc558d88582f5a4b is 50, key is test_row_0/C:col10/1732137366666/Put/seqid=0 2024-11-20T21:16:07,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742381_1557 (size=12301) 2024-11-20T21:16:07,650 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=345 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/65d48dedf6794638bc558d88582f5a4b 2024-11-20T21:16:07,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/c6bbddd1e5a345baa49cf35fff9d8cc2 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/c6bbddd1e5a345baa49cf35fff9d8cc2 2024-11-20T21:16:07,660 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/c6bbddd1e5a345baa49cf35fff9d8cc2, entries=150, sequenceid=345, filesize=12.0 K 2024-11-20T21:16:07,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/d927d40db64d4c2ebc36c5425329d7f4 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/d927d40db64d4c2ebc36c5425329d7f4 2024-11-20T21:16:07,664 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/d927d40db64d4c2ebc36c5425329d7f4, entries=150, sequenceid=345, filesize=12.0 K 2024-11-20T21:16:07,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/65d48dedf6794638bc558d88582f5a4b as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/65d48dedf6794638bc558d88582f5a4b 2024-11-20T21:16:07,673 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/65d48dedf6794638bc558d88582f5a4b, entries=150, sequenceid=345, filesize=12.0 K 2024-11-20T21:16:07,674 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 6c66f98c982744caf390af289e0ec6cf in 62ms, sequenceid=345, compaction requested=false 2024-11-20T21:16:07,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:07,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:07,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-20T21:16:07,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-20T21:16:07,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-20T21:16:07,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 996 msec 2024-11-20T21:16:07,680 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.0020 sec 2024-11-20T21:16:07,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:16:07,701 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c66f98c982744caf390af289e0ec6cf 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T21:16:07,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=A 2024-11-20T21:16:07,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:07,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=B 2024-11-20T21:16:07,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:07,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=C 2024-11-20T21:16:07,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:07,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/ba3444c73eb840b7bfeace9f35c23cab is 50, key is test_row_0/A:col10/1732137367685/Put/seqid=0 2024-11-20T21:16:07,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742382_1558 (size=12301) 2024-11-20T21:16:07,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-20T21:16:07,818 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:16:07,818 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:16:07,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-20T21:16:07,825 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:16:07,826 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:16:07,826 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:16:07,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T21:16:07,867 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732137367841, max timestamp=1732137367864Num uniques 859; 2024-11-20T21:16:07,868 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T21:16:07,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732137367841, max timestamp=1732137367864Num uniques 859; 2024-11-20T21:16:07,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1732137367841, max timestamp=1732137367864Num uniques 859; 2024-11-20T21:16:07,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T21:16:07,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-20T21:16:07,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:07,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137427914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:07,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:07,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137427915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:07,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T21:16:07,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:07,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137427933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:07,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:07,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:07,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:07,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:07,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137427934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:07,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:07,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137427935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:07,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:07,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:07,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:07,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:07,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:07,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:07,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-20T21:16:07,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:07,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:16:07,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:07,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:07,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:07,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:07,992 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/9f30037885f344038082f8a018844bd3 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/9f30037885f344038082f8a018844bd3 2024-11-20T21:16:07,997 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c66f98c982744caf390af289e0ec6cf/A of 6c66f98c982744caf390af289e0ec6cf into 9f30037885f344038082f8a018844bd3(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:07,997 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:07,997 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., storeName=6c66f98c982744caf390af289e0ec6cf/A, priority=13, startTime=1732137367545; duration=0sec 2024-11-20T21:16:07,997 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:07,997 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c66f98c982744caf390af289e0ec6cf:A 2024-11-20T21:16:07,998 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/d307f9bd497b450da53ce0baec25fd56 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/d307f9bd497b450da53ce0baec25fd56 2024-11-20T21:16:08,002 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c66f98c982744caf390af289e0ec6cf/B of 6c66f98c982744caf390af289e0ec6cf into d307f9bd497b450da53ce0baec25fd56(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:08,002 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:08,002 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., storeName=6c66f98c982744caf390af289e0ec6cf/B, priority=13, startTime=1732137367545; duration=0sec 2024-11-20T21:16:08,003 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:08,003 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c66f98c982744caf390af289e0ec6cf:B 2024-11-20T21:16:08,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:08,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137428040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:08,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:08,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:08,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137428047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:08,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:08,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:08,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137428057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:08,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:08,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:08,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137428058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:08,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:08,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:08,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137428058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:08,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,073 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:08,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/ba3444c73eb840b7bfeace9f35c23cab 2024-11-20T21:16:08,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/3f5957410b344028b001d3b35ad745ea is 50, key is test_row_0/B:col10/1732137367685/Put/seqid=0 2024-11-20T21:16:08,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-20T21:16:08,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:08,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:16:08,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:08,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:08,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:08,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:08,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T21:16:08,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742383_1559 (size=12301) 2024-11-20T21:16:08,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:08,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137428256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:08,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:08,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:08,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137428268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:08,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,277 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:08,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-20T21:16:08,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:08,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:16:08,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:08,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:08,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:08,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:08,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:08,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137428278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:08,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:08,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137428279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:08,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:08,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137428281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:08,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:08,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:08,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,293 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:08,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-20T21:16:08,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:08,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:16:08,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:08,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:08,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:08,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:08,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T21:16:08,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/3f5957410b344028b001d3b35ad745ea 2024-11-20T21:16:08,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/2622c40d02d84d9fb89a23c064c02dfa is 50, key is test_row_0/C:col10/1732137367685/Put/seqid=0 2024-11-20T21:16:08,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742384_1560 (size=12301) 2024-11-20T21:16:08,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=355 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/2622c40d02d84d9fb89a23c064c02dfa 2024-11-20T21:16:08,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/ba3444c73eb840b7bfeace9f35c23cab as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/ba3444c73eb840b7bfeace9f35c23cab 2024-11-20T21:16:08,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/ba3444c73eb840b7bfeace9f35c23cab, entries=150, sequenceid=355, filesize=12.0 K 2024-11-20T21:16:08,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/3f5957410b344028b001d3b35ad745ea as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/3f5957410b344028b001d3b35ad745ea 2024-11-20T21:16:08,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:08,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137428577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:08,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,586 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:08,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/3f5957410b344028b001d3b35ad745ea, entries=150, sequenceid=355, filesize=12.0 K 2024-11-20T21:16:08,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/2622c40d02d84d9fb89a23c064c02dfa as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/2622c40d02d84d9fb89a23c064c02dfa 2024-11-20T21:16:08,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/2622c40d02d84d9fb89a23c064c02dfa, entries=150, sequenceid=355, filesize=12.0 K 2024-11-20T21:16:08,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 6c66f98c982744caf390af289e0ec6cf in 891ms, sequenceid=355, compaction requested=true 2024-11-20T21:16:08,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:08,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c66f98c982744caf390af289e0ec6cf:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:16:08,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:08,592 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:16:08,592 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:16:08,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c66f98c982744caf390af289e0ec6cf:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:16:08,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:08,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c66f98c982744caf390af289e0ec6cf:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:16:08,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:16:08,593 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:16:08,593 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 6c66f98c982744caf390af289e0ec6cf/A is initiating minor compaction (all files) 2024-11-20T21:16:08,593 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c66f98c982744caf390af289e0ec6cf/A in TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:08,593 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/9f30037885f344038082f8a018844bd3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/c6bbddd1e5a345baa49cf35fff9d8cc2, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/ba3444c73eb840b7bfeace9f35c23cab] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp, totalSize=36.5 K 2024-11-20T21:16:08,593 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:16:08,593 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 6c66f98c982744caf390af289e0ec6cf/B is initiating minor compaction (all files) 2024-11-20T21:16:08,593 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f30037885f344038082f8a018844bd3, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1732137365418 2024-11-20T21:16:08,593 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c66f98c982744caf390af289e0ec6cf/B in TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:08,594 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/d307f9bd497b450da53ce0baec25fd56, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/d927d40db64d4c2ebc36c5425329d7f4, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/3f5957410b344028b001d3b35ad745ea] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp, totalSize=36.5 K 2024-11-20T21:16:08,596 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting d307f9bd497b450da53ce0baec25fd56, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1732137365418 2024-11-20T21:16:08,596 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6bbddd1e5a345baa49cf35fff9d8cc2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732137366658 2024-11-20T21:16:08,596 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting d927d40db64d4c2ebc36c5425329d7f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732137366658 2024-11-20T21:16:08,597 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f5957410b344028b001d3b35ad745ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732137367685 2024-11-20T21:16:08,597 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba3444c73eb840b7bfeace9f35c23cab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732137367685 2024-11-20T21:16:08,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-20T21:16:08,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:08,605 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 6c66f98c982744caf390af289e0ec6cf 3/3 column families, dataSize=140.89 KB heapSize=400.45 KB 2024-11-20T21:16:08,605 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c66f98c982744caf390af289e0ec6cf#B#compaction#476 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:08,606 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/74c2e7ced192470f995ecdb03ce486ef is 50, key is test_row_0/B:col10/1732137367685/Put/seqid=0 2024-11-20T21:16:08,608 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732137367908, max timestamp=1732137368599Num uniques 1068; 2024-11-20T21:16:08,608 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-20T21:16:08,610 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c66f98c982744caf390af289e0ec6cf#A#compaction#477 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:08,611 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/054fac080f914eb68dd9cb98b4172ffb is 50, key is test_row_0/A:col10/1732137367685/Put/seqid=0 2024-11-20T21:16:08,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:16:08,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=A 2024-11-20T21:16:08,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:16:08,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=B 2024-11-20T21:16:08,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:16:08,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T21:16:08,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=C 2024-11-20T21:16:08,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T21:16:08,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742386_1562 (size=12915) 2024-11-20T21:16:08,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742385_1561 (size=12915) 2024-11-20T21:16:08,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/24b0c3563f934b98ad75cedee6909d0a is 50, key is test_row_0/A:col10/1732137367932/Put/seqid=0 2024-11-20T21:16:08,648 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/74c2e7ced192470f995ecdb03ce486ef as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/74c2e7ced192470f995ecdb03ce486ef 2024-11-20T21:16:08,658 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c66f98c982744caf390af289e0ec6cf/B of 6c66f98c982744caf390af289e0ec6cf into 74c2e7ced192470f995ecdb03ce486ef(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:08,658 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:08,658 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., storeName=6c66f98c982744caf390af289e0ec6cf/B, priority=13, startTime=1732137368592; duration=0sec 2024-11-20T21:16:08,659 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:16:08,659 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c66f98c982744caf390af289e0ec6cf:B 2024-11-20T21:16:08,659 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:16:08,662 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:16:08,662 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 6c66f98c982744caf390af289e0ec6cf/C is initiating minor compaction (all files) 2024-11-20T21:16:08,662 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c66f98c982744caf390af289e0ec6cf/C in TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:08,662 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/8c8c14ea72b64fe99c8edb13f8bb3c23, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/65d48dedf6794638bc558d88582f5a4b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/2622c40d02d84d9fb89a23c064c02dfa] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp, totalSize=36.5 K 2024-11-20T21:16:08,663 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c8c14ea72b64fe99c8edb13f8bb3c23, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1732137365418 2024-11-20T21:16:08,663 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 65d48dedf6794638bc558d88582f5a4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=345, earliestPutTs=1732137366658 2024-11-20T21:16:08,664 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 2622c40d02d84d9fb89a23c064c02dfa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732137367685 2024-11-20T21:16:08,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742387_1563 (size=14741) 2024-11-20T21:16:08,679 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/24b0c3563f934b98ad75cedee6909d0a 2024-11-20T21:16:08,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:08,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137428659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:08,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:08,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137428660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:08,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:08,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137428663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:08,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:08,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137428664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:08,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:08,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:08,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,684 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:08,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:08,718 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c66f98c982744caf390af289e0ec6cf#C#compaction#479 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:08,719 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/4d40f348bd8a4e5b8f821b3a7b20de47 is 50, key is test_row_0/C:col10/1732137367685/Put/seqid=0 2024-11-20T21:16:08,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/6a04e92e21ce4a2183f566c1d8376774 is 50, key is test_row_0/B:col10/1732137367932/Put/seqid=0 2024-11-20T21:16:08,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742388_1564 (size=12915) 2024-11-20T21:16:08,731 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/4d40f348bd8a4e5b8f821b3a7b20de47 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/4d40f348bd8a4e5b8f821b3a7b20de47 2024-11-20T21:16:08,739 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c66f98c982744caf390af289e0ec6cf/C of 6c66f98c982744caf390af289e0ec6cf into 4d40f348bd8a4e5b8f821b3a7b20de47(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:08,739 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:08,739 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., storeName=6c66f98c982744caf390af289e0ec6cf/C, priority=13, startTime=1732137368592; duration=0sec 2024-11-20T21:16:08,739 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:08,739 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c66f98c982744caf390af289e0ec6cf:C 2024-11-20T21:16:08,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742389_1565 (size=12301) 2024-11-20T21:16:08,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:08,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137428788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:08,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:08,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:08,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137428789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:08,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:08,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:08,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137428789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:08,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:08,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137428789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:08,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:08,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:08,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:08,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T21:16:09,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:09,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137429007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:09,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137429007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:09,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:09,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137429009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:09,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:09,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:09,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:09,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:09,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137429009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:09,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:09,017 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:09,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:09,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:09,018 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:09,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:09,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:09,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:09,036 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/054fac080f914eb68dd9cb98b4172ffb as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/054fac080f914eb68dd9cb98b4172ffb 2024-11-20T21:16:09,043 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c66f98c982744caf390af289e0ec6cf/A of 6c66f98c982744caf390af289e0ec6cf into 054fac080f914eb68dd9cb98b4172ffb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:09,043 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:09,043 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., storeName=6c66f98c982744caf390af289e0ec6cf/A, priority=13, startTime=1732137368592; duration=0sec 2024-11-20T21:16:09,043 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:09,043 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c66f98c982744caf390af289e0ec6cf:A 2024-11-20T21:16:09,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:09,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137429098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:09,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:09,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:09,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:09,155 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/6a04e92e21ce4a2183f566c1d8376774 2024-11-20T21:16:09,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/f04a57a8fe2f4e54acef1cb7d166a27c is 50, key is test_row_0/C:col10/1732137367932/Put/seqid=0 2024-11-20T21:16:09,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742390_1566 (size=12301) 2024-11-20T21:16:09,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:09,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137429329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:09,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:09,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:09,334 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:09,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:09,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137429333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:09,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137429333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:09,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:09,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137429333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:09,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:09,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:09,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:09,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:09,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:09,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:09,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:09,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:09,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:09,638 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/f04a57a8fe2f4e54acef1cb7d166a27c 2024-11-20T21:16:09,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/24b0c3563f934b98ad75cedee6909d0a as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/24b0c3563f934b98ad75cedee6909d0a 2024-11-20T21:16:09,645 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/24b0c3563f934b98ad75cedee6909d0a, entries=200, sequenceid=395, filesize=14.4 K 2024-11-20T21:16:09,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/6a04e92e21ce4a2183f566c1d8376774 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/6a04e92e21ce4a2183f566c1d8376774 2024-11-20T21:16:09,648 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/6a04e92e21ce4a2183f566c1d8376774, entries=150, sequenceid=395, filesize=12.0 K 2024-11-20T21:16:09,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/f04a57a8fe2f4e54acef1cb7d166a27c as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/f04a57a8fe2f4e54acef1cb7d166a27c 2024-11-20T21:16:09,653 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/f04a57a8fe2f4e54acef1cb7d166a27c, entries=150, sequenceid=395, filesize=12.0 K 2024-11-20T21:16:09,654 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~302.31 KB/309568, currentSize=73.80 KB/75570 for 6c66f98c982744caf390af289e0ec6cf in 1048ms, sequenceid=395, compaction requested=false 2024-11-20T21:16:09,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:09,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:09,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-20T21:16:09,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-20T21:16:09,656 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-20T21:16:09,657 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8290 sec 2024-11-20T21:16:09,658 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.8390 sec 2024-11-20T21:16:09,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:16:09,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c66f98c982744caf390af289e0ec6cf 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-20T21:16:09,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=A 2024-11-20T21:16:09,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:09,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=B 2024-11-20T21:16:09,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:09,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=C 2024-11-20T21:16:09,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:09,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/99441e82988e4bb3b13144a30577c075 is 50, key is test_row_0/A:col10/1732137368656/Put/seqid=0 2024-11-20T21:16:09,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742391_1567 (size=12297) 2024-11-20T21:16:09,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/99441e82988e4bb3b13144a30577c075 2024-11-20T21:16:09,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/2ff59b220bf3433882fe65d1b0709668 is 50, key is test_row_0/B:col10/1732137368656/Put/seqid=0 2024-11-20T21:16:09,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742392_1568 (size=9857) 2024-11-20T21:16:09,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:09,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137429869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:09,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:09,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:09,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:09,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:09,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137429878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:09,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:09,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:09,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:09,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:09,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137429881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:09,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:09,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137429884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:09,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:09,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:09,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:09,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:09,894 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:09,895 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:09,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-20T21:16:09,977 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:16:09,978 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:16:09,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-20T21:16:09,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T21:16:09,981 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:16:09,983 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:16:09,983 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:16:09,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:09,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137429987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:09,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:09,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137429989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:09,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:09,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:09,993 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:09,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:09,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:09,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:10,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:10,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137429996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:10,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:10,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:10,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:10,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:10,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137429997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:10,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:10,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:10,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:10,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T21:16:10,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:10,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137430116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:10,127 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 0b16ab3f13b7,36987,1732137236536 for 'test_row_0' in 6c66f98c982744caf390af289e0ec6cf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2212 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T21:16:10,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:10,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:10,128 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:10,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T21:16:10,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:10,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:16:10,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:10,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:10,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:10,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:10,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:10,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137430197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:10,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:10,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137430197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:10,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:10,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:10,211 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:10,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:10,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:10,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:10,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:10,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137430220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:10,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:10,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137430221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:10,225 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:10,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/2ff59b220bf3433882fe65d1b0709668 2024-11-20T21:16:10,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/74e0e86c6b014ae5b5af17e5e67ec9c2 is 50, key is test_row_0/C:col10/1732137368656/Put/seqid=0 2024-11-20T21:16:10,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742393_1569 (size=9857) 2024-11-20T21:16:10,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T21:16:10,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:10,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:16:10,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:10,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:10,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:10,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:10,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T21:16:10,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T21:16:10,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:10,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:16:10,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:10,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:10,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:10,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:10,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:10,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137430517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:10,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:10,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137430517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:10,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:10,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:10,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:10,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:10,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:10,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:10,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:10,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137430526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:10,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:10,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:10,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:10,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:10,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137430537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:10,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:10,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:10,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:10,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T21:16:10,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:10,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:16:10,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:10,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:10,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:10,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:10,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T21:16:10,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/74e0e86c6b014ae5b5af17e5e67ec9c2 2024-11-20T21:16:10,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/99441e82988e4bb3b13144a30577c075 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/99441e82988e4bb3b13144a30577c075 2024-11-20T21:16:10,683 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/99441e82988e4bb3b13144a30577c075, entries=150, sequenceid=413, filesize=12.0 K 2024-11-20T21:16:10,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/2ff59b220bf3433882fe65d1b0709668 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/2ff59b220bf3433882fe65d1b0709668 2024-11-20T21:16:10,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/2ff59b220bf3433882fe65d1b0709668, entries=100, sequenceid=413, filesize=9.6 K 2024-11-20T21:16:10,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/74e0e86c6b014ae5b5af17e5e67ec9c2 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/74e0e86c6b014ae5b5af17e5e67ec9c2 2024-11-20T21:16:10,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/74e0e86c6b014ae5b5af17e5e67ec9c2, entries=100, sequenceid=413, filesize=9.6 K 2024-11-20T21:16:10,691 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 6c66f98c982744caf390af289e0ec6cf in 850ms, sequenceid=413, compaction requested=true 2024-11-20T21:16:10,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:10,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c66f98c982744caf390af289e0ec6cf:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:16:10,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:10,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c66f98c982744caf390af289e0ec6cf:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:16:10,691 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:16:10,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:10,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c66f98c982744caf390af289e0ec6cf:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:16:10,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:16:10,691 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:16:10,692 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39953 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:16:10,692 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:16:10,692 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 6c66f98c982744caf390af289e0ec6cf/A is initiating minor compaction (all files) 2024-11-20T21:16:10,692 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 6c66f98c982744caf390af289e0ec6cf/B is initiating minor compaction (all files) 2024-11-20T21:16:10,692 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c66f98c982744caf390af289e0ec6cf/A in TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:10,692 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c66f98c982744caf390af289e0ec6cf/B in TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:10,692 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/054fac080f914eb68dd9cb98b4172ffb, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/24b0c3563f934b98ad75cedee6909d0a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/99441e82988e4bb3b13144a30577c075] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp, totalSize=39.0 K 2024-11-20T21:16:10,692 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/74c2e7ced192470f995ecdb03ce486ef, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/6a04e92e21ce4a2183f566c1d8376774, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/2ff59b220bf3433882fe65d1b0709668] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp, totalSize=34.3 K 2024-11-20T21:16:10,692 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 74c2e7ced192470f995ecdb03ce486ef, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732137367685 2024-11-20T21:16:10,692 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 054fac080f914eb68dd9cb98b4172ffb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732137367685 2024-11-20T21:16:10,692 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a04e92e21ce4a2183f566c1d8376774, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732137367932 2024-11-20T21:16:10,692 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24b0c3563f934b98ad75cedee6909d0a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732137367908 2024-11-20T21:16:10,693 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ff59b220bf3433882fe65d1b0709668, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732137368656 2024-11-20T21:16:10,693 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99441e82988e4bb3b13144a30577c075, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732137368656 2024-11-20T21:16:10,699 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c66f98c982744caf390af289e0ec6cf#A#compaction#485 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:10,699 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/89020b96b3d349e1977c7aa6d85f7c42 is 50, key is test_row_0/A:col10/1732137368656/Put/seqid=0 2024-11-20T21:16:10,701 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c66f98c982744caf390af289e0ec6cf#B#compaction#486 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:10,702 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/384e88f374524f2bb4e7057176789f8b is 50, key is test_row_0/B:col10/1732137368656/Put/seqid=0 2024-11-20T21:16:10,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742394_1570 (size=13017) 2024-11-20T21:16:10,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742395_1571 (size=13017) 2024-11-20T21:16:10,709 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/384e88f374524f2bb4e7057176789f8b as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/384e88f374524f2bb4e7057176789f8b 2024-11-20T21:16:10,712 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c66f98c982744caf390af289e0ec6cf/B of 6c66f98c982744caf390af289e0ec6cf into 384e88f374524f2bb4e7057176789f8b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:10,713 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:10,713 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., storeName=6c66f98c982744caf390af289e0ec6cf/B, priority=13, startTime=1732137370691; duration=0sec 2024-11-20T21:16:10,713 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:16:10,713 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c66f98c982744caf390af289e0ec6cf:B 2024-11-20T21:16:10,713 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:16:10,713 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:16:10,714 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 6c66f98c982744caf390af289e0ec6cf/C is initiating minor compaction (all files) 2024-11-20T21:16:10,714 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c66f98c982744caf390af289e0ec6cf/C in TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:10,714 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/4d40f348bd8a4e5b8f821b3a7b20de47, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/f04a57a8fe2f4e54acef1cb7d166a27c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/74e0e86c6b014ae5b5af17e5e67ec9c2] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp, totalSize=34.3 K 2024-11-20T21:16:10,714 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d40f348bd8a4e5b8f821b3a7b20de47, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=355, earliestPutTs=1732137367685 2024-11-20T21:16:10,714 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting f04a57a8fe2f4e54acef1cb7d166a27c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1732137367932 2024-11-20T21:16:10,714 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 74e0e86c6b014ae5b5af17e5e67ec9c2, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732137368656 2024-11-20T21:16:10,720 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c66f98c982744caf390af289e0ec6cf#C#compaction#487 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:10,720 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/5b3ba8661fc94e8cae38ea03bf170a73 is 50, key is test_row_0/C:col10/1732137368656/Put/seqid=0 2024-11-20T21:16:10,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742396_1572 (size=13017) 2024-11-20T21:16:10,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-20T21:16:10,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:10,753 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 6c66f98c982744caf390af289e0ec6cf 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T21:16:10,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=A 2024-11-20T21:16:10,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:10,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=B 2024-11-20T21:16:10,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:10,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=C 2024-11-20T21:16:10,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:10,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/0561e768a4b0452898758584c419bef8 is 50, key is test_row_0/A:col10/1732137369866/Put/seqid=0 2024-11-20T21:16:10,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742397_1573 (size=12301) 2024-11-20T21:16:10,760 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/0561e768a4b0452898758584c419bef8 2024-11-20T21:16:10,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/72be61fc0589494c9eb55dbff8063e9b is 50, key is test_row_0/B:col10/1732137369866/Put/seqid=0 2024-11-20T21:16:10,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742398_1574 (size=12301) 2024-11-20T21:16:11,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:16:11,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:16:11,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:11,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137431083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:11,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:11,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137431086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:11,094 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:11,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,094 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:11,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:11,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137431091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:11,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:11,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137431093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:11,105 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:11,106 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/89020b96b3d349e1977c7aa6d85f7c42 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/89020b96b3d349e1977c7aa6d85f7c42 2024-11-20T21:16:11,110 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c66f98c982744caf390af289e0ec6cf/A of 6c66f98c982744caf390af289e0ec6cf into 89020b96b3d349e1977c7aa6d85f7c42(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:11,110 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:11,110 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., storeName=6c66f98c982744caf390af289e0ec6cf/A, priority=13, startTime=1732137370691; duration=0sec 2024-11-20T21:16:11,110 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:11,110 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c66f98c982744caf390af289e0ec6cf:A 2024-11-20T21:16:11,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T21:16:11,137 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/5b3ba8661fc94e8cae38ea03bf170a73 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/5b3ba8661fc94e8cae38ea03bf170a73 2024-11-20T21:16:11,142 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c66f98c982744caf390af289e0ec6cf/C of 6c66f98c982744caf390af289e0ec6cf into 5b3ba8661fc94e8cae38ea03bf170a73(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:11,142 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:11,142 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., storeName=6c66f98c982744caf390af289e0ec6cf/C, priority=13, startTime=1732137370691; duration=0sec 2024-11-20T21:16:11,142 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:11,142 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c66f98c982744caf390af289e0ec6cf:C 2024-11-20T21:16:11,170 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/72be61fc0589494c9eb55dbff8063e9b 2024-11-20T21:16:11,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/d1b75add15fc414fb2f3ce39d42bfe27 is 50, key is test_row_0/C:col10/1732137369866/Put/seqid=0 2024-11-20T21:16:11,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742399_1575 (size=12301) 2024-11-20T21:16:11,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137431197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:11,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:11,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137431197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:11,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:11,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,209 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:11,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:11,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137431207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:11,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,209 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:11,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:11,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137431217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:11,224 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,224 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,224 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:11,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:11,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137431419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:11,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:11,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:11,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137431421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:11,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:11,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137431421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:11,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:11,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,426 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:11,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:11,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137431443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:11,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,456 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,457 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:11,579 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=431 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/d1b75add15fc414fb2f3ce39d42bfe27 2024-11-20T21:16:11,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/0561e768a4b0452898758584c419bef8 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/0561e768a4b0452898758584c419bef8 2024-11-20T21:16:11,586 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/0561e768a4b0452898758584c419bef8, entries=150, sequenceid=431, filesize=12.0 K 2024-11-20T21:16:11,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/72be61fc0589494c9eb55dbff8063e9b as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/72be61fc0589494c9eb55dbff8063e9b 2024-11-20T21:16:11,590 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/72be61fc0589494c9eb55dbff8063e9b, entries=150, sequenceid=431, filesize=12.0 K 2024-11-20T21:16:11,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/d1b75add15fc414fb2f3ce39d42bfe27 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/d1b75add15fc414fb2f3ce39d42bfe27 2024-11-20T21:16:11,593 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/d1b75add15fc414fb2f3ce39d42bfe27, entries=150, sequenceid=431, filesize=12.0 K 2024-11-20T21:16:11,594 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 6c66f98c982744caf390af289e0ec6cf in 841ms, sequenceid=431, compaction requested=false 2024-11-20T21:16:11,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:11,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:11,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-20T21:16:11,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-20T21:16:11,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-20T21:16:11,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6130 sec 2024-11-20T21:16:11,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.6210 sec 2024-11-20T21:16:11,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:16:11,743 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c66f98c982744caf390af289e0ec6cf 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-20T21:16:11,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=A 2024-11-20T21:16:11,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:11,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=B 2024-11-20T21:16:11,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:11,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=C 2024-11-20T21:16:11,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:11,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/c2067d4965534d3280bf855ffcf25e04 is 50, key is test_row_0/A:col10/1732137371741/Put/seqid=0 2024-11-20T21:16:11,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742400_1576 (size=14741) 2024-11-20T21:16:11,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:11,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137431769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:11,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:11,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137431771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:11,781 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,782 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:11,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:11,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:11,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137431775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:11,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:11,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:11,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137431776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:11,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:11,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137431887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:11,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:11,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137431887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:11,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:11,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:11,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137431897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:11,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:11,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137431898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:11,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:11,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:11,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:12,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137432107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:12,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137432107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:12,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:12,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137432107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:12,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:12,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:12,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137432107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:12,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:12,123 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,123 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,123 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:12,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-20T21:16:12,127 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:16:12,128 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:16:12,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-20T21:16:12,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T21:16:12,130 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:16:12,131 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:16:12,131 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:16:12,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/c2067d4965534d3280bf855ffcf25e04 2024-11-20T21:16:12,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:12,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56714 deadline: 1732137432146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:12,158 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 0b16ab3f13b7,36987,1732137236536 for 'test_row_0' in 6c66f98c982744caf390af289e0ec6cf of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T21:16:12,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:12,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/7bf6de5909864bc89a1bc6047e0facb3 is 50, key is test_row_0/B:col10/1732137371741/Put/seqid=0 2024-11-20T21:16:12,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742401_1577 (size=12301) 2024-11-20T21:16:12,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/7bf6de5909864bc89a1bc6047e0facb3 2024-11-20T21:16:12,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/4fe81694904349168173109d357d7392 is 50, key is test_row_0/C:col10/1732137371741/Put/seqid=0 2024-11-20T21:16:12,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742402_1578 (size=12301) 2024-11-20T21:16:12,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/4fe81694904349168173109d357d7392 2024-11-20T21:16:12,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/c2067d4965534d3280bf855ffcf25e04 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/c2067d4965534d3280bf855ffcf25e04 2024-11-20T21:16:12,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/c2067d4965534d3280bf855ffcf25e04, entries=200, sequenceid=452, filesize=14.4 K 2024-11-20T21:16:12,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/7bf6de5909864bc89a1bc6047e0facb3 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/7bf6de5909864bc89a1bc6047e0facb3 2024-11-20T21:16:12,187 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/7bf6de5909864bc89a1bc6047e0facb3, entries=150, sequenceid=452, filesize=12.0 K 2024-11-20T21:16:12,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/4fe81694904349168173109d357d7392 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/4fe81694904349168173109d357d7392 2024-11-20T21:16:12,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/4fe81694904349168173109d357d7392, entries=150, sequenceid=452, filesize=12.0 K 2024-11-20T21:16:12,191 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6c66f98c982744caf390af289e0ec6cf in 448ms, sequenceid=452, compaction requested=true 2024-11-20T21:16:12,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:12,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c66f98c982744caf390af289e0ec6cf:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:16:12,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:12,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c66f98c982744caf390af289e0ec6cf:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:16:12,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:12,192 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:16:12,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c66f98c982744caf390af289e0ec6cf:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:16:12,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:16:12,192 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:16:12,193 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:16:12,193 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:16:12,193 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 6c66f98c982744caf390af289e0ec6cf/A is initiating minor compaction (all files) 2024-11-20T21:16:12,193 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 6c66f98c982744caf390af289e0ec6cf/B is initiating minor compaction (all files) 2024-11-20T21:16:12,193 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c66f98c982744caf390af289e0ec6cf/B in TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:12,193 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c66f98c982744caf390af289e0ec6cf/A in TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:12,193 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/384e88f374524f2bb4e7057176789f8b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/72be61fc0589494c9eb55dbff8063e9b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/7bf6de5909864bc89a1bc6047e0facb3] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp, totalSize=36.7 K 2024-11-20T21:16:12,193 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/89020b96b3d349e1977c7aa6d85f7c42, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/0561e768a4b0452898758584c419bef8, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/c2067d4965534d3280bf855ffcf25e04] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp, totalSize=39.1 K 2024-11-20T21:16:12,193 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 384e88f374524f2bb4e7057176789f8b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732137368599 2024-11-20T21:16:12,193 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89020b96b3d349e1977c7aa6d85f7c42, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732137368599 2024-11-20T21:16:12,194 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 72be61fc0589494c9eb55dbff8063e9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1732137369866 2024-11-20T21:16:12,194 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0561e768a4b0452898758584c419bef8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1732137369866 2024-11-20T21:16:12,194 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bf6de5909864bc89a1bc6047e0facb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1732137371741 2024-11-20T21:16:12,194 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2067d4965534d3280bf855ffcf25e04, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1732137371085 2024-11-20T21:16:12,207 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c66f98c982744caf390af289e0ec6cf#B#compaction#494 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:12,207 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/277bc802eb5444c79af05d20ac790119 is 50, key is test_row_0/B:col10/1732137371741/Put/seqid=0 2024-11-20T21:16:12,212 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c66f98c982744caf390af289e0ec6cf#A#compaction#495 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:12,213 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/fee91703c83c4f92b228aa215a41fa31 is 50, key is test_row_0/A:col10/1732137371741/Put/seqid=0 2024-11-20T21:16:12,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742403_1579 (size=13119) 2024-11-20T21:16:12,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T21:16:12,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742404_1580 (size=13119) 2024-11-20T21:16:12,250 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/fee91703c83c4f92b228aa215a41fa31 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/fee91703c83c4f92b228aa215a41fa31 2024-11-20T21:16:12,256 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c66f98c982744caf390af289e0ec6cf/A of 6c66f98c982744caf390af289e0ec6cf into fee91703c83c4f92b228aa215a41fa31(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:12,256 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:12,256 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., storeName=6c66f98c982744caf390af289e0ec6cf/A, priority=13, startTime=1732137372192; duration=0sec 2024-11-20T21:16:12,256 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:16:12,256 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c66f98c982744caf390af289e0ec6cf:A 2024-11-20T21:16:12,256 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:16:12,257 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:16:12,257 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 6c66f98c982744caf390af289e0ec6cf/C is initiating minor compaction (all files) 2024-11-20T21:16:12,257 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c66f98c982744caf390af289e0ec6cf/C in TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:12,257 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/5b3ba8661fc94e8cae38ea03bf170a73, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/d1b75add15fc414fb2f3ce39d42bfe27, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/4fe81694904349168173109d357d7392] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp, totalSize=36.7 K 2024-11-20T21:16:12,259 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b3ba8661fc94e8cae38ea03bf170a73, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1732137368599 2024-11-20T21:16:12,260 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1b75add15fc414fb2f3ce39d42bfe27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=431, earliestPutTs=1732137369866 2024-11-20T21:16:12,260 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fe81694904349168173109d357d7392, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1732137371741 2024-11-20T21:16:12,267 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c66f98c982744caf390af289e0ec6cf#C#compaction#496 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:12,267 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/346c5bd92062432fac01e7698aba5df2 is 50, key is test_row_0/C:col10/1732137371741/Put/seqid=0 2024-11-20T21:16:12,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742405_1581 (size=13119) 2024-11-20T21:16:12,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-20T21:16:12,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:12,284 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 6c66f98c982744caf390af289e0ec6cf 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-20T21:16:12,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=A 2024-11-20T21:16:12,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:12,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=B 2024-11-20T21:16:12,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:12,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=C 2024-11-20T21:16:12,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:12,290 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/346c5bd92062432fac01e7698aba5df2 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/346c5bd92062432fac01e7698aba5df2 2024-11-20T21:16:12,300 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c66f98c982744caf390af289e0ec6cf/C of 6c66f98c982744caf390af289e0ec6cf into 346c5bd92062432fac01e7698aba5df2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:12,301 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:12,301 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., storeName=6c66f98c982744caf390af289e0ec6cf/C, priority=13, startTime=1732137372192; duration=0sec 2024-11-20T21:16:12,301 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:12,301 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c66f98c982744caf390af289e0ec6cf:C 2024-11-20T21:16:12,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/914c246b4c464c8ba8415e83da778a54 is 50, key is test_row_0/A:col10/1732137371773/Put/seqid=0 2024-11-20T21:16:12,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742406_1582 (size=9857) 2024-11-20T21:16:12,331 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/914c246b4c464c8ba8415e83da778a54 2024-11-20T21:16:12,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/a5c893376e7c4fbd90b0b78bdddb443e is 50, key is test_row_0/B:col10/1732137371773/Put/seqid=0 2024-11-20T21:16:12,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742407_1583 (size=9857) 2024-11-20T21:16:12,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:16:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:16:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T21:16:12,504 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137432488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:12,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:12,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:12,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137432506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:12,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:12,522 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:12,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137432506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:12,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:12,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,523 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137432506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:12,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,523 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:12,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137432616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:12,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:12,631 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/277bc802eb5444c79af05d20ac790119 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/277bc802eb5444c79af05d20ac790119 2024-11-20T21:16:12,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137432627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:12,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:12,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137432627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:12,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:12,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:12,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137432627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:12,633 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:12,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:12,636 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c66f98c982744caf390af289e0ec6cf/B of 6c66f98c982744caf390af289e0ec6cf into 277bc802eb5444c79af05d20ac790119(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:12,636 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:12,636 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., storeName=6c66f98c982744caf390af289e0ec6cf/B, priority=13, startTime=1732137372192; duration=0sec 2024-11-20T21:16:12,636 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:12,636 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c66f98c982744caf390af289e0ec6cf:B 2024-11-20T21:16:12,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T21:16:12,759 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/a5c893376e7c4fbd90b0b78bdddb443e 2024-11-20T21:16:12,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/5d51bef2de0147ffaaa6410887d526b2 is 50, key is test_row_0/C:col10/1732137371773/Put/seqid=0 2024-11-20T21:16:12,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742408_1584 (size=9857) 2024-11-20T21:16:12,770 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=466 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/5d51bef2de0147ffaaa6410887d526b2 2024-11-20T21:16:12,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/914c246b4c464c8ba8415e83da778a54 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/914c246b4c464c8ba8415e83da778a54 2024-11-20T21:16:12,781 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/914c246b4c464c8ba8415e83da778a54, entries=100, sequenceid=466, filesize=9.6 K 2024-11-20T21:16:12,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/a5c893376e7c4fbd90b0b78bdddb443e as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/a5c893376e7c4fbd90b0b78bdddb443e 2024-11-20T21:16:12,784 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/a5c893376e7c4fbd90b0b78bdddb443e, entries=100, sequenceid=466, filesize=9.6 K 2024-11-20T21:16:12,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/5d51bef2de0147ffaaa6410887d526b2 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/5d51bef2de0147ffaaa6410887d526b2 2024-11-20T21:16:12,788 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/5d51bef2de0147ffaaa6410887d526b2, entries=100, sequenceid=466, filesize=9.6 K 2024-11-20T21:16:12,789 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 6c66f98c982744caf390af289e0ec6cf in 505ms, sequenceid=466, compaction requested=false 2024-11-20T21:16:12,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:12,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:12,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-20T21:16:12,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-20T21:16:12,794 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-20T21:16:12,794 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 659 msec 2024-11-20T21:16:12,795 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 666 msec 2024-11-20T21:16:12,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:16:12,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c66f98c982744caf390af289e0ec6cf 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-20T21:16:12,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=A 2024-11-20T21:16:12,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:16:12,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=B 2024-11-20T21:16:12,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:16:12,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=C 2024-11-20T21:16:12,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:16:12,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/9f30941db0734970b96c8a15a1c2596c is 50, key is test_row_0/A:col10/1732137372831/Put/seqid=0 2024-11-20T21:16:12,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742409_1585 (size=14741) 2024-11-20T21:16:12,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/9f30941db0734970b96c8a15a1c2596c 2024-11-20T21:16:12,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/d9b6d19f902445fdb4716a608b58c6a1 is 50, key is test_row_0/B:col10/1732137372831/Put/seqid=0 2024-11-20T21:16:12,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742410_1586 (size=12301) 2024-11-20T21:16:12,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/d9b6d19f902445fdb4716a608b58c6a1 2024-11-20T21:16:12,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/c3476f49c444473fb31a822b4b09ea1d is 50, key is test_row_0/C:col10/1732137372831/Put/seqid=0 2024-11-20T21:16:12,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:12,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742411_1587 (size=12301) 2024-11-20T21:16:12,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137432859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:12,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=489 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/c3476f49c444473fb31a822b4b09ea1d 2024-11-20T21:16:12,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/9f30941db0734970b96c8a15a1c2596c as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/9f30941db0734970b96c8a15a1c2596c 2024-11-20T21:16:12,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,869 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:12,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/9f30941db0734970b96c8a15a1c2596c, entries=200, sequenceid=489, filesize=14.4 K 2024-11-20T21:16:12,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/d9b6d19f902445fdb4716a608b58c6a1 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/d9b6d19f902445fdb4716a608b58c6a1 2024-11-20T21:16:12,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:12,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137432861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:12,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:12,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137432862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:12,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:12,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137432862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:12,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/d9b6d19f902445fdb4716a608b58c6a1, entries=150, sequenceid=489, filesize=12.0 K 2024-11-20T21:16:12,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:12,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,876 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:12,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:12,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/c3476f49c444473fb31a822b4b09ea1d as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/c3476f49c444473fb31a822b4b09ea1d 2024-11-20T21:16:12,876 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:12,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/c3476f49c444473fb31a822b4b09ea1d, entries=150, sequenceid=489, filesize=12.0 K 2024-11-20T21:16:12,881 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 6c66f98c982744caf390af289e0ec6cf in 49ms, sequenceid=489, compaction requested=true 2024-11-20T21:16:12,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:12,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c66f98c982744caf390af289e0ec6cf:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:16:12,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:12,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c66f98c982744caf390af289e0ec6cf:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:16:12,881 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:16:12,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:12,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c66f98c982744caf390af289e0ec6cf:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:16:12,881 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:16:12,881 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:16:12,882 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:16:12,882 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37717 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:16:12,882 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 6c66f98c982744caf390af289e0ec6cf/B is initiating minor compaction (all files) 2024-11-20T21:16:12,882 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 6c66f98c982744caf390af289e0ec6cf/A is initiating minor compaction (all files) 2024-11-20T21:16:12,882 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c66f98c982744caf390af289e0ec6cf/B in TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:12,882 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c66f98c982744caf390af289e0ec6cf/A in TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:12,883 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/277bc802eb5444c79af05d20ac790119, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/a5c893376e7c4fbd90b0b78bdddb443e, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/d9b6d19f902445fdb4716a608b58c6a1] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp, totalSize=34.5 K 2024-11-20T21:16:12,883 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/fee91703c83c4f92b228aa215a41fa31, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/914c246b4c464c8ba8415e83da778a54, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/9f30941db0734970b96c8a15a1c2596c] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp, totalSize=36.8 K 2024-11-20T21:16:12,883 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 277bc802eb5444c79af05d20ac790119, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1732137371741 2024-11-20T21:16:12,884 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting fee91703c83c4f92b228aa215a41fa31, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1732137371741 2024-11-20T21:16:12,884 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting a5c893376e7c4fbd90b0b78bdddb443e, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1732137371773 2024-11-20T21:16:12,884 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 914c246b4c464c8ba8415e83da778a54, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1732137371773 2024-11-20T21:16:12,884 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f30941db0734970b96c8a15a1c2596c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1732137372486 2024-11-20T21:16:12,887 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting d9b6d19f902445fdb4716a608b58c6a1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1732137372486 2024-11-20T21:16:12,902 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c66f98c982744caf390af289e0ec6cf#B#compaction#503 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:12,902 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/8caf4bfacafb4eeb9b7565aa732c4e07 is 50, key is test_row_0/B:col10/1732137372831/Put/seqid=0 2024-11-20T21:16:12,903 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c66f98c982744caf390af289e0ec6cf#A#compaction#504 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:12,904 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/7bb89c1bbc344caba5130d3dd3e4048e is 50, key is test_row_0/A:col10/1732137372831/Put/seqid=0 2024-11-20T21:16:12,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742412_1588 (size=13221) 2024-11-20T21:16:12,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742413_1589 (size=13221) 2024-11-20T21:16:12,914 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/7bb89c1bbc344caba5130d3dd3e4048e as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/7bb89c1bbc344caba5130d3dd3e4048e 2024-11-20T21:16:12,918 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c66f98c982744caf390af289e0ec6cf/A of 6c66f98c982744caf390af289e0ec6cf into 7bb89c1bbc344caba5130d3dd3e4048e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:12,918 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:12,918 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., storeName=6c66f98c982744caf390af289e0ec6cf/A, priority=13, startTime=1732137372881; duration=0sec 2024-11-20T21:16:12,918 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:16:12,918 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c66f98c982744caf390af289e0ec6cf:A 2024-11-20T21:16:12,918 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:16:12,919 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:16:12,919 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 6c66f98c982744caf390af289e0ec6cf/C is initiating minor compaction (all files) 2024-11-20T21:16:12,919 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c66f98c982744caf390af289e0ec6cf/C in TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:12,919 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/346c5bd92062432fac01e7698aba5df2, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/5d51bef2de0147ffaaa6410887d526b2, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/c3476f49c444473fb31a822b4b09ea1d] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp, totalSize=34.5 K 2024-11-20T21:16:12,920 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 346c5bd92062432fac01e7698aba5df2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=452, earliestPutTs=1732137371741 2024-11-20T21:16:12,920 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d51bef2de0147ffaaa6410887d526b2, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=466, earliestPutTs=1732137371773 2024-11-20T21:16:12,920 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3476f49c444473fb31a822b4b09ea1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1732137372486 2024-11-20T21:16:12,927 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c66f98c982744caf390af289e0ec6cf#C#compaction#505 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:12,928 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/035b40f723c14c63b28dfcc91c5befb4 is 50, key is test_row_0/C:col10/1732137372831/Put/seqid=0 2024-11-20T21:16:12,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742414_1590 (size=13221) 2024-11-20T21:16:12,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:16:12,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c66f98c982744caf390af289e0ec6cf 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-20T21:16:12,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=A 2024-11-20T21:16:12,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:12,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=B 2024-11-20T21:16:12,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:12,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=C 2024-11-20T21:16:12,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:13,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/f3671c4cba16432eae5095a338e1bf86 is 50, key is test_row_0/A:col10/1732137372980/Put/seqid=0 2024-11-20T21:16:13,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742415_1591 (size=17181) 2024-11-20T21:16:13,014 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/f3671c4cba16432eae5095a338e1bf86 2024-11-20T21:16:13,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/801c8f520ace4793be35f171c06e9f5c is 50, key is test_row_0/B:col10/1732137372980/Put/seqid=0 2024-11-20T21:16:13,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742416_1592 (size=12301) 2024-11-20T21:16:13,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/801c8f520ace4793be35f171c06e9f5c 2024-11-20T21:16:13,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/82fca2445e8f4927909ce5ed35a0d58e is 50, key is test_row_0/C:col10/1732137372980/Put/seqid=0 2024-11-20T21:16:13,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742417_1593 (size=12301) 2024-11-20T21:16:13,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:13,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137433042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:13,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:13,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:13,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137433042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:13,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:13,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:13,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137433045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:13,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,056 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,057 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:13,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:13,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137433050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:13,066 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,066 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,066 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:13,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:13,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137433167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:13,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:13,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137433167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:13,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:13,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:13,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137433167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:13,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:13,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:13,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:13,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137433177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:13,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:13,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-20T21:16:13,267 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:16:13,267 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:16:13,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-20T21:16:13,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T21:16:13,269 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:16:13,269 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:16:13,270 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:16:13,316 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/8caf4bfacafb4eeb9b7565aa732c4e07 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/8caf4bfacafb4eeb9b7565aa732c4e07 2024-11-20T21:16:13,321 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c66f98c982744caf390af289e0ec6cf/B of 6c66f98c982744caf390af289e0ec6cf into 8caf4bfacafb4eeb9b7565aa732c4e07(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:13,321 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:13,321 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., storeName=6c66f98c982744caf390af289e0ec6cf/B, priority=13, startTime=1732137372881; duration=0sec 2024-11-20T21:16:13,321 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:13,321 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c66f98c982744caf390af289e0ec6cf:B 2024-11-20T21:16:13,349 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/035b40f723c14c63b28dfcc91c5befb4 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/035b40f723c14c63b28dfcc91c5befb4 2024-11-20T21:16:13,353 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c66f98c982744caf390af289e0ec6cf/C of 6c66f98c982744caf390af289e0ec6cf into 035b40f723c14c63b28dfcc91c5befb4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:13,353 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:13,353 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., storeName=6c66f98c982744caf390af289e0ec6cf/C, priority=13, startTime=1732137372881; duration=0sec 2024-11-20T21:16:13,353 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:13,353 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c66f98c982744caf390af289e0ec6cf:C 2024-11-20T21:16:13,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T21:16:13,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:13,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137433387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:13,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:13,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137433387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:13,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,397 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:13,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:13,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137433387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:13,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:13,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:13,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137433389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:13,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,398 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:13,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-20T21:16:13,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:13,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:16:13,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:13,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:13,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:13,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:13,449 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=503 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/82fca2445e8f4927909ce5ed35a0d58e 2024-11-20T21:16:13,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/f3671c4cba16432eae5095a338e1bf86 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/f3671c4cba16432eae5095a338e1bf86 2024-11-20T21:16:13,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/f3671c4cba16432eae5095a338e1bf86, entries=250, sequenceid=503, filesize=16.8 K 2024-11-20T21:16:13,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/801c8f520ace4793be35f171c06e9f5c as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/801c8f520ace4793be35f171c06e9f5c 2024-11-20T21:16:13,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/801c8f520ace4793be35f171c06e9f5c, entries=150, sequenceid=503, filesize=12.0 K 2024-11-20T21:16:13,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/82fca2445e8f4927909ce5ed35a0d58e as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/82fca2445e8f4927909ce5ed35a0d58e 2024-11-20T21:16:13,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/82fca2445e8f4927909ce5ed35a0d58e, entries=150, sequenceid=503, filesize=12.0 K 2024-11-20T21:16:13,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 6c66f98c982744caf390af289e0ec6cf in 573ms, sequenceid=503, compaction requested=false 2024-11-20T21:16:13,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:13,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-20T21:16:13,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:13,581 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 6c66f98c982744caf390af289e0ec6cf 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-20T21:16:13,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=A 2024-11-20T21:16:13,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:13,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=B 2024-11-20T21:16:13,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:13,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=C 2024-11-20T21:16:13,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:13,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T21:16:13,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/53653e73caa74ba885d6c5e310f82bb7 is 50, key is test_row_0/A:col10/1732137373045/Put/seqid=0 2024-11-20T21:16:13,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742418_1594 (size=12301) 2024-11-20T21:16:13,621 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/53653e73caa74ba885d6c5e310f82bb7 2024-11-20T21:16:13,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/959595bbdb864eb4b41539a9640c75cb is 50, key is test_row_0/B:col10/1732137373045/Put/seqid=0 2024-11-20T21:16:13,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742419_1595 (size=12301) 2024-11-20T21:16:13,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:16:13,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. as already flushing 2024-11-20T21:16:13,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:13,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137433728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:13,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:13,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137433729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:13,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:13,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137433730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:13,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,742 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:13,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:13,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137433732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:13,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,747 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:13,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:13,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137433847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:13,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:13,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:13,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137433847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:13,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:13,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137433847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:13,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:13,859 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:13,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:13,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137433857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:13,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:13,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:13,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T21:16:14,038 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/959595bbdb864eb4b41539a9640c75cb 2024-11-20T21:16:14,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/33d823e460da44babe89d880dc50d715 is 50, key is test_row_0/C:col10/1732137373045/Put/seqid=0 2024-11-20T21:16:14,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742420_1596 (size=12301) 2024-11-20T21:16:14,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137434067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:14,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:14,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:14,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:14,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137434067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:14,079 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:14,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:14,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137434067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:14,080 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:14,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137434068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:14,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:14,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:14,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:14,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:14,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:14,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56788 deadline: 1732137434387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:14,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:14,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56734 deadline: 1732137434387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:14,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:14,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56744 deadline: 1732137434387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:14,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:14,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56758 deadline: 1732137434387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:14,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:14,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:14,401 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:14,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:14,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:14,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:14,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:14,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 , the old value is region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:14,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:14,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:14,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:14,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=2 because the exception is null or not the one we care about 2024-11-20T21:16:14,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T21:16:14,457 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=524 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/33d823e460da44babe89d880dc50d715 2024-11-20T21:16:14,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/53653e73caa74ba885d6c5e310f82bb7 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/53653e73caa74ba885d6c5e310f82bb7 2024-11-20T21:16:14,471 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/53653e73caa74ba885d6c5e310f82bb7, entries=150, sequenceid=524, filesize=12.0 K 2024-11-20T21:16:14,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/959595bbdb864eb4b41539a9640c75cb as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/959595bbdb864eb4b41539a9640c75cb 2024-11-20T21:16:14,485 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/959595bbdb864eb4b41539a9640c75cb, entries=150, sequenceid=524, filesize=12.0 K 2024-11-20T21:16:14,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/33d823e460da44babe89d880dc50d715 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/33d823e460da44babe89d880dc50d715 2024-11-20T21:16:14,503 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/33d823e460da44babe89d880dc50d715, entries=150, sequenceid=524, filesize=12.0 K 2024-11-20T21:16:14,503 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6c66f98c982744caf390af289e0ec6cf in 922ms, sequenceid=524, compaction requested=true 2024-11-20T21:16:14,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:14,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:14,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-20T21:16:14,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-20T21:16:14,510 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-20T21:16:14,510 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2380 sec 2024-11-20T21:16:14,512 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.2440 sec 2024-11-20T21:16:14,891 INFO [Thread-2217 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2217. 2024-11-20T21:16:14,891 DEBUG [Thread-2217 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:16:14,891 DEBUG [Thread-2217 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:14,891 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:16:14,891 DEBUG [Thread-2217 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:14,894 INFO [Thread-2219 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2219. 2024-11-20T21:16:14,894 DEBUG [Thread-2219 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:16:14,894 DEBUG [Thread-2219 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:14,894 DEBUG [Thread-2219 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:14,895 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:16:14,897 INFO [Thread-2213 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2213. 2024-11-20T21:16:14,897 DEBUG [Thread-2213 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:16:14,897 DEBUG [Thread-2213 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:14,897 DEBUG [Thread-2213 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:14,897 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:16:14,899 INFO [Thread-2221 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2221. 2024-11-20T21:16:14,899 DEBUG [Thread-2221 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:16:14,899 DEBUG [Thread-2221 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:14,899 DEBUG [Thread-2221 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:14,899 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:16:14,905 INFO [Thread-2215 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2215. 2024-11-20T21:16:14,905 DEBUG [Thread-2215 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:16:14,905 DEBUG [Thread-2215 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:14,905 DEBUG [Thread-2215 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:14,905 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:16:14,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:16:14,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c66f98c982744caf390af289e0ec6cf 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-20T21:16:14,909 INFO [Thread-2210 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2210. 2024-11-20T21:16:14,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=A 2024-11-20T21:16:14,909 DEBUG [Thread-2210 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:16:14,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:14,909 DEBUG [Thread-2210 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:14,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=B 2024-11-20T21:16:14,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:14,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=C 2024-11-20T21:16:14,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:14,909 DEBUG [Thread-2210 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:14,909 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:16:14,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/c033c312796c495cb912cafb9ef93dd9 is 50, key is test_row_0/A:col10/1732137373716/Put/seqid=0 2024-11-20T21:16:14,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742421_1597 (size=12301) 2024-11-20T21:16:14,919 INFO [Thread-2204 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2204. 2024-11-20T21:16:14,919 DEBUG [Thread-2204 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:16:14,919 DEBUG [Thread-2204 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:14,919 DEBUG [Thread-2204 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:14,919 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:16:14,921 INFO [Thread-2206 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2206. 2024-11-20T21:16:14,921 DEBUG [Thread-2206 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:16:14,921 DEBUG [Thread-2206 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:14,921 DEBUG [Thread-2206 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:14,921 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:16:14,923 INFO [Thread-2208 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2208. 2024-11-20T21:16:14,923 DEBUG [Thread-2208 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:16:14,923 DEBUG [Thread-2208 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:14,923 DEBUG [Thread-2208 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:14,923 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:16:15,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=538 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/c033c312796c495cb912cafb9ef93dd9 2024-11-20T21:16:15,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/824d789579e043aa85341aa406abc61b is 50, key is test_row_0/B:col10/1732137373716/Put/seqid=0 2024-11-20T21:16:15,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742422_1598 (size=12301) 2024-11-20T21:16:15,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-20T21:16:15,417 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:16:15,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=538 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/824d789579e043aa85341aa406abc61b 2024-11-20T21:16:15,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/4cf686a4a1d94556950408aae1b9c5db is 50, key is test_row_0/C:col10/1732137373716/Put/seqid=0 2024-11-20T21:16:15,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742423_1599 (size=12301) 2024-11-20T21:16:16,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=538 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/4cf686a4a1d94556950408aae1b9c5db 2024-11-20T21:16:16,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/c033c312796c495cb912cafb9ef93dd9 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/c033c312796c495cb912cafb9ef93dd9 2024-11-20T21:16:16,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/c033c312796c495cb912cafb9ef93dd9, entries=150, sequenceid=538, filesize=12.0 K 2024-11-20T21:16:16,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/824d789579e043aa85341aa406abc61b as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/824d789579e043aa85341aa406abc61b 2024-11-20T21:16:16,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/824d789579e043aa85341aa406abc61b, entries=150, sequenceid=538, filesize=12.0 K 2024-11-20T21:16:16,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/4cf686a4a1d94556950408aae1b9c5db as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/4cf686a4a1d94556950408aae1b9c5db 2024-11-20T21:16:16,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/4cf686a4a1d94556950408aae1b9c5db, entries=150, sequenceid=538, filesize=12.0 K 2024-11-20T21:16:16,158 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=20.13 KB/20610 for 6c66f98c982744caf390af289e0ec6cf in 1248ms, sequenceid=538, compaction requested=true 2024-11-20T21:16:16,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:16,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c66f98c982744caf390af289e0ec6cf:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:16:16,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:16,158 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:16:16,158 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:16:16,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c66f98c982744caf390af289e0ec6cf:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:16:16,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:16,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c66f98c982744caf390af289e0ec6cf:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:16:16,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:16:16,159 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:16:16,159 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:16:16,159 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 6c66f98c982744caf390af289e0ec6cf/B is initiating minor compaction (all files) 2024-11-20T21:16:16,159 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 6c66f98c982744caf390af289e0ec6cf/A is initiating minor compaction (all files) 2024-11-20T21:16:16,159 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c66f98c982744caf390af289e0ec6cf/B in TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:16,159 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c66f98c982744caf390af289e0ec6cf/A in TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:16,159 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/8caf4bfacafb4eeb9b7565aa732c4e07, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/801c8f520ace4793be35f171c06e9f5c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/959595bbdb864eb4b41539a9640c75cb, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/824d789579e043aa85341aa406abc61b] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp, totalSize=48.9 K 2024-11-20T21:16:16,159 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/7bb89c1bbc344caba5130d3dd3e4048e, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/f3671c4cba16432eae5095a338e1bf86, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/53653e73caa74ba885d6c5e310f82bb7, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/c033c312796c495cb912cafb9ef93dd9] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp, totalSize=53.7 K 2024-11-20T21:16:16,159 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8caf4bfacafb4eeb9b7565aa732c4e07, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1732137372486 2024-11-20T21:16:16,159 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bb89c1bbc344caba5130d3dd3e4048e, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1732137372486 2024-11-20T21:16:16,159 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 801c8f520ace4793be35f171c06e9f5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=503, earliestPutTs=1732137372859 2024-11-20T21:16:16,159 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3671c4cba16432eae5095a338e1bf86, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=503, earliestPutTs=1732137372842 2024-11-20T21:16:16,160 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 959595bbdb864eb4b41539a9640c75cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=524, earliestPutTs=1732137373039 2024-11-20T21:16:16,160 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53653e73caa74ba885d6c5e310f82bb7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=524, earliestPutTs=1732137373039 2024-11-20T21:16:16,160 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 824d789579e043aa85341aa406abc61b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=538, earliestPutTs=1732137373716 2024-11-20T21:16:16,160 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting c033c312796c495cb912cafb9ef93dd9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=538, earliestPutTs=1732137373716 2024-11-20T21:16:16,166 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c66f98c982744caf390af289e0ec6cf#A#compaction#516 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:16,166 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c66f98c982744caf390af289e0ec6cf#B#compaction#515 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:16,167 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/8dce591ba1074c57b79086fa7821ac2a is 50, key is test_row_0/A:col10/1732137373716/Put/seqid=0 2024-11-20T21:16:16,167 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/7d840c5b8eb64bf692b25aec4c32a4b0 is 50, key is test_row_0/B:col10/1732137373716/Put/seqid=0 2024-11-20T21:16:16,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742424_1600 (size=13357) 2024-11-20T21:16:16,176 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/7d840c5b8eb64bf692b25aec4c32a4b0 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/7d840c5b8eb64bf692b25aec4c32a4b0 2024-11-20T21:16:16,178 INFO [Thread-2202 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2202. 2024-11-20T21:16:16,178 DEBUG [Thread-2202 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:16:16,178 DEBUG [Thread-2202 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:16,178 DEBUG [Thread-2202 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:16,178 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:16:16,179 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T21:16:16,179 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-20T21:16:16,179 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-20T21:16:16,179 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 104 2024-11-20T21:16:16,179 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 110 2024-11-20T21:16:16,179 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-11-20T21:16:16,179 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T21:16:16,179 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T21:16:16,179 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 858 2024-11-20T21:16:16,179 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2573 rows 2024-11-20T21:16:16,179 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 862 2024-11-20T21:16:16,179 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2582 rows 2024-11-20T21:16:16,179 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 865 2024-11-20T21:16:16,179 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2594 rows 2024-11-20T21:16:16,179 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 866 2024-11-20T21:16:16,179 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2594 rows 2024-11-20T21:16:16,179 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 863 2024-11-20T21:16:16,179 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2588 rows 2024-11-20T21:16:16,179 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T21:16:16,179 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T21:16:16,179 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:16,179 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:16,180 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:16:16,180 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-20T21:16:16,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742425_1601 (size=13357) 2024-11-20T21:16:16,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T21:16:16,182 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6c66f98c982744caf390af289e0ec6cf/B of 6c66f98c982744caf390af289e0ec6cf into 7d840c5b8eb64bf692b25aec4c32a4b0(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:16,182 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:16,182 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., storeName=6c66f98c982744caf390af289e0ec6cf/B, priority=12, startTime=1732137376158; duration=0sec 2024-11-20T21:16:16,182 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:16:16,182 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c66f98c982744caf390af289e0ec6cf:B 2024-11-20T21:16:16,182 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:16:16,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-20T21:16:16,183 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:16:16,183 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 6c66f98c982744caf390af289e0ec6cf/C is initiating minor compaction (all files) 2024-11-20T21:16:16,184 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c66f98c982744caf390af289e0ec6cf/C in TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:16,184 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/035b40f723c14c63b28dfcc91c5befb4, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/82fca2445e8f4927909ce5ed35a0d58e, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/33d823e460da44babe89d880dc50d715, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/4cf686a4a1d94556950408aae1b9c5db] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp, totalSize=48.9 K 2024-11-20T21:16:16,184 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732137376183"}]},"ts":"1732137376183"} 2024-11-20T21:16:16,184 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 035b40f723c14c63b28dfcc91c5befb4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=489, earliestPutTs=1732137372486 2024-11-20T21:16:16,184 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 82fca2445e8f4927909ce5ed35a0d58e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=503, earliestPutTs=1732137372859 2024-11-20T21:16:16,185 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 33d823e460da44babe89d880dc50d715, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=524, earliestPutTs=1732137373039 2024-11-20T21:16:16,185 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T21:16:16,185 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cf686a4a1d94556950408aae1b9c5db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=538, earliestPutTs=1732137373716 2024-11-20T21:16:16,185 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T21:16:16,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T21:16:16,187 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c66f98c982744caf390af289e0ec6cf, UNASSIGN}] 2024-11-20T21:16:16,188 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c66f98c982744caf390af289e0ec6cf, UNASSIGN 2024-11-20T21:16:16,189 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=6c66f98c982744caf390af289e0ec6cf, regionState=CLOSING, regionLocation=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:16,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c66f98c982744caf390af289e0ec6cf, UNASSIGN because future has completed 2024-11-20T21:16:16,191 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T21:16:16,191 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536}] 2024-11-20T21:16:16,196 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c66f98c982744caf390af289e0ec6cf#C#compaction#517 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:16,196 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/ffa9d5bfb4ad4d1c89fe1f47b3878e9b is 50, key is test_row_0/C:col10/1732137373716/Put/seqid=0 2024-11-20T21:16:16,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742426_1602 (size=13357) 2024-11-20T21:16:16,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-20T21:16:16,343 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:16:16,343 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T21:16:16,343 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing 6c66f98c982744caf390af289e0ec6cf, disabling compactions & flushes 2024-11-20T21:16:16,343 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:16,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-20T21:16:16,587 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/8dce591ba1074c57b79086fa7821ac2a as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/8dce591ba1074c57b79086fa7821ac2a 2024-11-20T21:16:16,601 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6c66f98c982744caf390af289e0ec6cf/A of 6c66f98c982744caf390af289e0ec6cf into 8dce591ba1074c57b79086fa7821ac2a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:16,601 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:16,601 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., storeName=6c66f98c982744caf390af289e0ec6cf/A, priority=12, startTime=1732137376158; duration=0sec 2024-11-20T21:16:16,601 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:16,601 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c66f98c982744caf390af289e0ec6cf:A 2024-11-20T21:16:16,613 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/ffa9d5bfb4ad4d1c89fe1f47b3878e9b as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/ffa9d5bfb4ad4d1c89fe1f47b3878e9b 2024-11-20T21:16:16,620 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6c66f98c982744caf390af289e0ec6cf/C of 6c66f98c982744caf390af289e0ec6cf into ffa9d5bfb4ad4d1c89fe1f47b3878e9b(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:16,620 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c66f98c982744caf390af289e0ec6cf: 2024-11-20T21:16:16,620 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., storeName=6c66f98c982744caf390af289e0ec6cf/C, priority=12, startTime=1732137376158; duration=0sec 2024-11-20T21:16:16,620 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:16,620 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c66f98c982744caf390af289e0ec6cf:C 2024-11-20T21:16:16,620 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:16,620 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:16,620 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. after waiting 0 ms 2024-11-20T21:16:16,620 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:16,620 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(2902): Flushing 6c66f98c982744caf390af289e0ec6cf 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-20T21:16:16,621 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=A 2024-11-20T21:16:16,621 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:16,621 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=B 2024-11-20T21:16:16,621 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:16,621 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c66f98c982744caf390af289e0ec6cf, store=C 2024-11-20T21:16:16,621 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:16,626 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/7a7b5c74ca6041bd93a18fdac26ba63a is 50, key is test_row_0/A:col10/1732137376177/Put/seqid=0 2024-11-20T21:16:16,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742427_1603 (size=9857) 2024-11-20T21:16:16,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-20T21:16:17,033 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/7a7b5c74ca6041bd93a18fdac26ba63a 2024-11-20T21:16:17,038 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/28399e13f59c495d88a8f1990fce1b67 is 50, key is test_row_0/B:col10/1732137376177/Put/seqid=0 2024-11-20T21:16:17,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742428_1604 (size=9857) 2024-11-20T21:16:17,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-20T21:16:17,442 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/28399e13f59c495d88a8f1990fce1b67 2024-11-20T21:16:17,447 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/bb1b61f87bfd44cd839a9c0059c7b63c is 50, key is test_row_0/C:col10/1732137376177/Put/seqid=0 2024-11-20T21:16:17,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742429_1605 (size=9857) 2024-11-20T21:16:17,851 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/bb1b61f87bfd44cd839a9c0059c7b63c 2024-11-20T21:16:17,856 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/A/7a7b5c74ca6041bd93a18fdac26ba63a as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/7a7b5c74ca6041bd93a18fdac26ba63a 2024-11-20T21:16:17,860 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/7a7b5c74ca6041bd93a18fdac26ba63a, entries=100, sequenceid=548, filesize=9.6 K 2024-11-20T21:16:17,860 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/B/28399e13f59c495d88a8f1990fce1b67 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/28399e13f59c495d88a8f1990fce1b67 2024-11-20T21:16:17,864 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/28399e13f59c495d88a8f1990fce1b67, entries=100, sequenceid=548, filesize=9.6 K 2024-11-20T21:16:17,865 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/.tmp/C/bb1b61f87bfd44cd839a9c0059c7b63c as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/bb1b61f87bfd44cd839a9c0059c7b63c 2024-11-20T21:16:17,868 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/bb1b61f87bfd44cd839a9c0059c7b63c, entries=100, sequenceid=548, filesize=9.6 K 2024-11-20T21:16:17,869 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 6c66f98c982744caf390af289e0ec6cf in 1249ms, sequenceid=548, compaction requested=false 2024-11-20T21:16:17,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/fe4c3aafc97c4cd7b69f629d26da4a23, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/35e6431dc25b4a15adccc62a7c715ac4, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/04ee58eb8fee40239a9eff6a01f199d7, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/49e3e8bfb60f41d3a8f4b7a7a5cd2784, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/75b4fd02a2514c92a3244c79079f90b2, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/1ccd6d31abf84bc4b43cc67994f4d200, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/c13f890c94c4418e9dfc39cb24cc4d3b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/2d49867e21574bf5ab3e5ea026827a23, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/3b0a3ccb07c34a739a9c2917a5ef323b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/da432cb55aa04bf68dd01b01eb11f61a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/ab581d7384bf4c7a9a0187ccc0f815d5, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/8e02a7ac44f147c8b19cc8e90fb21a75, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/ddac5a8d79bf40f78940d78ddb2e8524, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/e4af9cab5d1a460aa7b4d3d9d8182df3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/ff47b0d8b124421eb7e0db3e397cb480, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/9f30037885f344038082f8a018844bd3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/c6bbddd1e5a345baa49cf35fff9d8cc2, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/054fac080f914eb68dd9cb98b4172ffb, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/ba3444c73eb840b7bfeace9f35c23cab, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/24b0c3563f934b98ad75cedee6909d0a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/89020b96b3d349e1977c7aa6d85f7c42, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/99441e82988e4bb3b13144a30577c075, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/0561e768a4b0452898758584c419bef8, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/c2067d4965534d3280bf855ffcf25e04, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/fee91703c83c4f92b228aa215a41fa31, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/914c246b4c464c8ba8415e83da778a54, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/9f30941db0734970b96c8a15a1c2596c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/7bb89c1bbc344caba5130d3dd3e4048e, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/f3671c4cba16432eae5095a338e1bf86, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/53653e73caa74ba885d6c5e310f82bb7, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/c033c312796c495cb912cafb9ef93dd9] to archive 2024-11-20T21:16:17,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T21:16:17,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/fe4c3aafc97c4cd7b69f629d26da4a23 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/fe4c3aafc97c4cd7b69f629d26da4a23 2024-11-20T21:16:17,873 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/35e6431dc25b4a15adccc62a7c715ac4 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/35e6431dc25b4a15adccc62a7c715ac4 2024-11-20T21:16:17,874 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/04ee58eb8fee40239a9eff6a01f199d7 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/04ee58eb8fee40239a9eff6a01f199d7 2024-11-20T21:16:17,885 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/49e3e8bfb60f41d3a8f4b7a7a5cd2784 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/49e3e8bfb60f41d3a8f4b7a7a5cd2784 2024-11-20T21:16:17,886 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/75b4fd02a2514c92a3244c79079f90b2 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/75b4fd02a2514c92a3244c79079f90b2 2024-11-20T21:16:17,892 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/1ccd6d31abf84bc4b43cc67994f4d200 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/1ccd6d31abf84bc4b43cc67994f4d200 2024-11-20T21:16:17,894 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/c13f890c94c4418e9dfc39cb24cc4d3b to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/c13f890c94c4418e9dfc39cb24cc4d3b 2024-11-20T21:16:17,895 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/2d49867e21574bf5ab3e5ea026827a23 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/2d49867e21574bf5ab3e5ea026827a23 2024-11-20T21:16:17,896 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/3b0a3ccb07c34a739a9c2917a5ef323b to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/3b0a3ccb07c34a739a9c2917a5ef323b 2024-11-20T21:16:17,897 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/da432cb55aa04bf68dd01b01eb11f61a to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/da432cb55aa04bf68dd01b01eb11f61a 2024-11-20T21:16:17,899 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/ab581d7384bf4c7a9a0187ccc0f815d5 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/ab581d7384bf4c7a9a0187ccc0f815d5 2024-11-20T21:16:17,900 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/8e02a7ac44f147c8b19cc8e90fb21a75 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/8e02a7ac44f147c8b19cc8e90fb21a75 2024-11-20T21:16:17,901 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/ddac5a8d79bf40f78940d78ddb2e8524 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/ddac5a8d79bf40f78940d78ddb2e8524 2024-11-20T21:16:17,902 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/e4af9cab5d1a460aa7b4d3d9d8182df3 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/e4af9cab5d1a460aa7b4d3d9d8182df3 2024-11-20T21:16:17,903 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/ff47b0d8b124421eb7e0db3e397cb480 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/ff47b0d8b124421eb7e0db3e397cb480 2024-11-20T21:16:17,905 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/9f30037885f344038082f8a018844bd3 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/9f30037885f344038082f8a018844bd3 2024-11-20T21:16:17,906 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/c6bbddd1e5a345baa49cf35fff9d8cc2 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/c6bbddd1e5a345baa49cf35fff9d8cc2 2024-11-20T21:16:17,907 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/054fac080f914eb68dd9cb98b4172ffb to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/054fac080f914eb68dd9cb98b4172ffb 2024-11-20T21:16:17,908 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/ba3444c73eb840b7bfeace9f35c23cab to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/ba3444c73eb840b7bfeace9f35c23cab 2024-11-20T21:16:17,909 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/24b0c3563f934b98ad75cedee6909d0a to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/24b0c3563f934b98ad75cedee6909d0a 2024-11-20T21:16:17,910 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/89020b96b3d349e1977c7aa6d85f7c42 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/89020b96b3d349e1977c7aa6d85f7c42 2024-11-20T21:16:17,911 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/99441e82988e4bb3b13144a30577c075 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/99441e82988e4bb3b13144a30577c075 2024-11-20T21:16:17,912 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/0561e768a4b0452898758584c419bef8 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/0561e768a4b0452898758584c419bef8 2024-11-20T21:16:17,914 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/c2067d4965534d3280bf855ffcf25e04 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/c2067d4965534d3280bf855ffcf25e04 2024-11-20T21:16:17,915 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/fee91703c83c4f92b228aa215a41fa31 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/fee91703c83c4f92b228aa215a41fa31 2024-11-20T21:16:17,916 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/914c246b4c464c8ba8415e83da778a54 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/914c246b4c464c8ba8415e83da778a54 2024-11-20T21:16:17,917 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/9f30941db0734970b96c8a15a1c2596c to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/9f30941db0734970b96c8a15a1c2596c 2024-11-20T21:16:17,918 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/7bb89c1bbc344caba5130d3dd3e4048e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/7bb89c1bbc344caba5130d3dd3e4048e 2024-11-20T21:16:17,919 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/f3671c4cba16432eae5095a338e1bf86 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/f3671c4cba16432eae5095a338e1bf86 2024-11-20T21:16:17,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/53653e73caa74ba885d6c5e310f82bb7 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/53653e73caa74ba885d6c5e310f82bb7 2024-11-20T21:16:17,920 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/c033c312796c495cb912cafb9ef93dd9 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/c033c312796c495cb912cafb9ef93dd9 2024-11-20T21:16:17,922 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/9893e32ac671412b9e433d69c35f921b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/713a1ce5d04a410baf849d281f638277, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/507e10e21e6e4a14bd39e918105c72fa, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/67f6b210352e406598d7f3180f7003f3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/9dbd9f120f89495aaa144c3d473a66ae, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/86ebd761223e41cfa48e6db5b280e65a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/3a0f43a3218f446ba3724862d0181b52, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/54ad62d07aa044bd90bbe1c675f114d3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/ec939cd02ac0451baea9af8d95aceccc, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/9f7ec3b4a422400ab9bc2ea9e291864d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/fd8cc43102db4aeba015a62d6c7d6f4c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/64c20dee55894fdfa83be0841aa97b09, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/f862725547d64ddd8df2ea758ef0c829, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/dbcf395525054cd391668992c9e61acc, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/d307f9bd497b450da53ce0baec25fd56, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/141629e1a111430bb6a7d492aae9f0c6, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/d927d40db64d4c2ebc36c5425329d7f4, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/74c2e7ced192470f995ecdb03ce486ef, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/3f5957410b344028b001d3b35ad745ea, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/6a04e92e21ce4a2183f566c1d8376774, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/384e88f374524f2bb4e7057176789f8b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/2ff59b220bf3433882fe65d1b0709668, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/72be61fc0589494c9eb55dbff8063e9b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/277bc802eb5444c79af05d20ac790119, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/7bf6de5909864bc89a1bc6047e0facb3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/a5c893376e7c4fbd90b0b78bdddb443e, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/8caf4bfacafb4eeb9b7565aa732c4e07, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/d9b6d19f902445fdb4716a608b58c6a1, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/801c8f520ace4793be35f171c06e9f5c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/959595bbdb864eb4b41539a9640c75cb, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/824d789579e043aa85341aa406abc61b] to archive 2024-11-20T21:16:17,923 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T21:16:17,924 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/9893e32ac671412b9e433d69c35f921b to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/9893e32ac671412b9e433d69c35f921b 2024-11-20T21:16:17,925 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/713a1ce5d04a410baf849d281f638277 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/713a1ce5d04a410baf849d281f638277 2024-11-20T21:16:17,926 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/507e10e21e6e4a14bd39e918105c72fa to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/507e10e21e6e4a14bd39e918105c72fa 2024-11-20T21:16:17,927 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/67f6b210352e406598d7f3180f7003f3 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/67f6b210352e406598d7f3180f7003f3 2024-11-20T21:16:17,928 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/9dbd9f120f89495aaa144c3d473a66ae to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/9dbd9f120f89495aaa144c3d473a66ae 2024-11-20T21:16:17,929 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/86ebd761223e41cfa48e6db5b280e65a to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/86ebd761223e41cfa48e6db5b280e65a 2024-11-20T21:16:17,930 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/3a0f43a3218f446ba3724862d0181b52 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/3a0f43a3218f446ba3724862d0181b52 2024-11-20T21:16:17,931 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/54ad62d07aa044bd90bbe1c675f114d3 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/54ad62d07aa044bd90bbe1c675f114d3 2024-11-20T21:16:17,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/ec939cd02ac0451baea9af8d95aceccc to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/ec939cd02ac0451baea9af8d95aceccc 2024-11-20T21:16:17,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/9f7ec3b4a422400ab9bc2ea9e291864d to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/9f7ec3b4a422400ab9bc2ea9e291864d 2024-11-20T21:16:17,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/fd8cc43102db4aeba015a62d6c7d6f4c to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/fd8cc43102db4aeba015a62d6c7d6f4c 2024-11-20T21:16:17,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/64c20dee55894fdfa83be0841aa97b09 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/64c20dee55894fdfa83be0841aa97b09 2024-11-20T21:16:17,937 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/f862725547d64ddd8df2ea758ef0c829 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/f862725547d64ddd8df2ea758ef0c829 2024-11-20T21:16:17,938 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/dbcf395525054cd391668992c9e61acc to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/dbcf395525054cd391668992c9e61acc 2024-11-20T21:16:17,938 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/d307f9bd497b450da53ce0baec25fd56 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/d307f9bd497b450da53ce0baec25fd56 2024-11-20T21:16:17,939 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/141629e1a111430bb6a7d492aae9f0c6 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/141629e1a111430bb6a7d492aae9f0c6 2024-11-20T21:16:17,940 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/d927d40db64d4c2ebc36c5425329d7f4 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/d927d40db64d4c2ebc36c5425329d7f4 2024-11-20T21:16:17,941 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/74c2e7ced192470f995ecdb03ce486ef to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/74c2e7ced192470f995ecdb03ce486ef 2024-11-20T21:16:17,942 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/3f5957410b344028b001d3b35ad745ea to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/3f5957410b344028b001d3b35ad745ea 2024-11-20T21:16:17,943 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/6a04e92e21ce4a2183f566c1d8376774 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/6a04e92e21ce4a2183f566c1d8376774 2024-11-20T21:16:17,944 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/384e88f374524f2bb4e7057176789f8b to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/384e88f374524f2bb4e7057176789f8b 2024-11-20T21:16:17,945 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/2ff59b220bf3433882fe65d1b0709668 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/2ff59b220bf3433882fe65d1b0709668 2024-11-20T21:16:17,946 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/72be61fc0589494c9eb55dbff8063e9b to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/72be61fc0589494c9eb55dbff8063e9b 2024-11-20T21:16:17,947 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/277bc802eb5444c79af05d20ac790119 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/277bc802eb5444c79af05d20ac790119 2024-11-20T21:16:17,948 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/7bf6de5909864bc89a1bc6047e0facb3 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/7bf6de5909864bc89a1bc6047e0facb3 2024-11-20T21:16:17,949 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/a5c893376e7c4fbd90b0b78bdddb443e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/a5c893376e7c4fbd90b0b78bdddb443e 2024-11-20T21:16:17,952 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/8caf4bfacafb4eeb9b7565aa732c4e07 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/8caf4bfacafb4eeb9b7565aa732c4e07 2024-11-20T21:16:17,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/d9b6d19f902445fdb4716a608b58c6a1 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/d9b6d19f902445fdb4716a608b58c6a1 2024-11-20T21:16:17,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/801c8f520ace4793be35f171c06e9f5c to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/801c8f520ace4793be35f171c06e9f5c 2024-11-20T21:16:17,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/959595bbdb864eb4b41539a9640c75cb to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/959595bbdb864eb4b41539a9640c75cb 2024-11-20T21:16:17,958 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/824d789579e043aa85341aa406abc61b to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/824d789579e043aa85341aa406abc61b 2024-11-20T21:16:17,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/10875b29b3ef4838b4ac181225e27b62, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/afc83d782713497cbd9dbd2c48acf402, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/9f77bfa8c6194db1acd19de9765df72e, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/766bc78b98f0445a88088b04aff78c83, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/29b7437681524c3db9cbc75b610f1f1a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/52ae69b05cd54a3b9ace18b7739f17db, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/b1e791265fb4486380e2c8cf378966df, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/18fad13f0fc949f8a5bdf0ccfc8cce73, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/6fc07c45cd3146d1a0f6011e08400353, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/422dee926cfa45e0b12d3e03f3157e06, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/8d24c58d5ada4691a329761e58dd63c5, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/ef0fdfe7e9134506baf0e33430b668f4, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/813688808fc548feaacf66a2dae86f8e, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/92c13df3cdf94c61ba3d557cffdd087e, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/8c8c14ea72b64fe99c8edb13f8bb3c23, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/8879cdd7bbab4494b417dd0e5b5336e7, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/65d48dedf6794638bc558d88582f5a4b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/4d40f348bd8a4e5b8f821b3a7b20de47, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/2622c40d02d84d9fb89a23c064c02dfa, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/f04a57a8fe2f4e54acef1cb7d166a27c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/5b3ba8661fc94e8cae38ea03bf170a73, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/74e0e86c6b014ae5b5af17e5e67ec9c2, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/d1b75add15fc414fb2f3ce39d42bfe27, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/346c5bd92062432fac01e7698aba5df2, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/4fe81694904349168173109d357d7392, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/5d51bef2de0147ffaaa6410887d526b2, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/035b40f723c14c63b28dfcc91c5befb4, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/c3476f49c444473fb31a822b4b09ea1d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/82fca2445e8f4927909ce5ed35a0d58e, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/33d823e460da44babe89d880dc50d715, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/4cf686a4a1d94556950408aae1b9c5db] to archive 2024-11-20T21:16:17,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T21:16:17,962 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/10875b29b3ef4838b4ac181225e27b62 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/10875b29b3ef4838b4ac181225e27b62 2024-11-20T21:16:17,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/afc83d782713497cbd9dbd2c48acf402 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/afc83d782713497cbd9dbd2c48acf402 2024-11-20T21:16:17,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/9f77bfa8c6194db1acd19de9765df72e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/9f77bfa8c6194db1acd19de9765df72e 2024-11-20T21:16:17,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/766bc78b98f0445a88088b04aff78c83 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/766bc78b98f0445a88088b04aff78c83 2024-11-20T21:16:17,968 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/29b7437681524c3db9cbc75b610f1f1a to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/29b7437681524c3db9cbc75b610f1f1a 2024-11-20T21:16:17,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/52ae69b05cd54a3b9ace18b7739f17db to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/52ae69b05cd54a3b9ace18b7739f17db 2024-11-20T21:16:17,970 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/b1e791265fb4486380e2c8cf378966df to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/b1e791265fb4486380e2c8cf378966df 2024-11-20T21:16:17,971 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/18fad13f0fc949f8a5bdf0ccfc8cce73 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/18fad13f0fc949f8a5bdf0ccfc8cce73 2024-11-20T21:16:17,972 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/6fc07c45cd3146d1a0f6011e08400353 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/6fc07c45cd3146d1a0f6011e08400353 2024-11-20T21:16:17,975 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/422dee926cfa45e0b12d3e03f3157e06 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/422dee926cfa45e0b12d3e03f3157e06 2024-11-20T21:16:17,976 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/8d24c58d5ada4691a329761e58dd63c5 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/8d24c58d5ada4691a329761e58dd63c5 2024-11-20T21:16:17,978 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/ef0fdfe7e9134506baf0e33430b668f4 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/ef0fdfe7e9134506baf0e33430b668f4 2024-11-20T21:16:17,979 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/813688808fc548feaacf66a2dae86f8e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/813688808fc548feaacf66a2dae86f8e 2024-11-20T21:16:17,981 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/92c13df3cdf94c61ba3d557cffdd087e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/92c13df3cdf94c61ba3d557cffdd087e 2024-11-20T21:16:17,983 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/8c8c14ea72b64fe99c8edb13f8bb3c23 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/8c8c14ea72b64fe99c8edb13f8bb3c23 2024-11-20T21:16:17,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/8879cdd7bbab4494b417dd0e5b5336e7 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/8879cdd7bbab4494b417dd0e5b5336e7 2024-11-20T21:16:17,986 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/65d48dedf6794638bc558d88582f5a4b to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/65d48dedf6794638bc558d88582f5a4b 2024-11-20T21:16:17,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/4d40f348bd8a4e5b8f821b3a7b20de47 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/4d40f348bd8a4e5b8f821b3a7b20de47 2024-11-20T21:16:17,989 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/2622c40d02d84d9fb89a23c064c02dfa to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/2622c40d02d84d9fb89a23c064c02dfa 2024-11-20T21:16:17,990 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/f04a57a8fe2f4e54acef1cb7d166a27c to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/f04a57a8fe2f4e54acef1cb7d166a27c 2024-11-20T21:16:17,992 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/5b3ba8661fc94e8cae38ea03bf170a73 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/5b3ba8661fc94e8cae38ea03bf170a73 2024-11-20T21:16:17,994 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/74e0e86c6b014ae5b5af17e5e67ec9c2 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/74e0e86c6b014ae5b5af17e5e67ec9c2 2024-11-20T21:16:17,995 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/d1b75add15fc414fb2f3ce39d42bfe27 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/d1b75add15fc414fb2f3ce39d42bfe27 2024-11-20T21:16:17,996 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/346c5bd92062432fac01e7698aba5df2 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/346c5bd92062432fac01e7698aba5df2 2024-11-20T21:16:17,997 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/4fe81694904349168173109d357d7392 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/4fe81694904349168173109d357d7392 2024-11-20T21:16:17,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/5d51bef2de0147ffaaa6410887d526b2 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/5d51bef2de0147ffaaa6410887d526b2 2024-11-20T21:16:18,000 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/035b40f723c14c63b28dfcc91c5befb4 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/035b40f723c14c63b28dfcc91c5befb4 2024-11-20T21:16:18,001 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/c3476f49c444473fb31a822b4b09ea1d to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/c3476f49c444473fb31a822b4b09ea1d 2024-11-20T21:16:18,002 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/82fca2445e8f4927909ce5ed35a0d58e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/82fca2445e8f4927909ce5ed35a0d58e 2024-11-20T21:16:18,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/33d823e460da44babe89d880dc50d715 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/33d823e460da44babe89d880dc50d715 2024-11-20T21:16:18,004 DEBUG [StoreCloser-TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/4cf686a4a1d94556950408aae1b9c5db to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/4cf686a4a1d94556950408aae1b9c5db 2024-11-20T21:16:18,013 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/recovered.edits/551.seqid, newMaxSeqId=551, maxSeqId=1 2024-11-20T21:16:18,014 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. 2024-11-20T21:16:18,014 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for 6c66f98c982744caf390af289e0ec6cf: Waiting for close lock at 1732137376343Running coprocessor pre-close hooks at 1732137376343Disabling compacts and flushes for region at 1732137376343Disabling writes for close at 1732137376620 (+277 ms)Obtaining lock to block concurrent updates at 1732137376620Preparing flush snapshotting stores in 6c66f98c982744caf390af289e0ec6cf at 1732137376620Finished memstore snapshotting TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1732137376621 (+1 ms)Flushing stores of TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf. at 1732137376621Flushing 6c66f98c982744caf390af289e0ec6cf/A: creating writer at 1732137376621Flushing 6c66f98c982744caf390af289e0ec6cf/A: appending metadata at 1732137376626 (+5 ms)Flushing 6c66f98c982744caf390af289e0ec6cf/A: closing flushed file at 1732137376626Flushing 6c66f98c982744caf390af289e0ec6cf/B: creating writer at 1732137377036 (+410 ms)Flushing 6c66f98c982744caf390af289e0ec6cf/B: appending metadata at 1732137377038 (+2 ms)Flushing 6c66f98c982744caf390af289e0ec6cf/B: closing flushed file at 1732137377038Flushing 6c66f98c982744caf390af289e0ec6cf/C: creating writer at 1732137377445 (+407 ms)Flushing 6c66f98c982744caf390af289e0ec6cf/C: appending metadata at 1732137377447 (+2 ms)Flushing 6c66f98c982744caf390af289e0ec6cf/C: closing flushed file at 1732137377447Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@18979992: reopening flushed file at 1732137377855 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b788474: reopening flushed file at 1732137377860 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66d25e8: reopening flushed file at 1732137377864 (+4 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 6c66f98c982744caf390af289e0ec6cf in 1249ms, sequenceid=548, compaction requested=false at 1732137377869 (+5 ms)Writing region close event to WAL at 1732137378009 (+140 ms)Running coprocessor post-close hooks at 1732137378014 (+5 ms)Closed at 1732137378014 2024-11-20T21:16:18,017 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed 6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:16:18,017 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=6c66f98c982744caf390af289e0ec6cf, regionState=CLOSED 2024-11-20T21:16:18,019 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 because future has completed 2024-11-20T21:16:18,022 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-20T21:16:18,022 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure 6c66f98c982744caf390af289e0ec6cf, server=0b16ab3f13b7,36987,1732137236536 in 1.8290 sec 2024-11-20T21:16:18,026 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-20T21:16:18,026 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c66f98c982744caf390af289e0ec6cf, UNASSIGN in 1.8350 sec 2024-11-20T21:16:18,029 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-20T21:16:18,029 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8410 sec 2024-11-20T21:16:18,030 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732137378030"}]},"ts":"1732137378030"} 2024-11-20T21:16:18,032 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T21:16:18,032 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T21:16:18,034 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8530 sec 2024-11-20T21:16:18,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-20T21:16:18,327 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:16:18,327 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-20T21:16:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:16:18,329 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:16:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-20T21:16:18,330 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=143, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:16:18,332 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:16:18,334 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A, FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B, FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C, FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/recovered.edits] 2024-11-20T21:16:18,336 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/7a7b5c74ca6041bd93a18fdac26ba63a to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/7a7b5c74ca6041bd93a18fdac26ba63a 2024-11-20T21:16:18,337 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/8dce591ba1074c57b79086fa7821ac2a to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/A/8dce591ba1074c57b79086fa7821ac2a 2024-11-20T21:16:18,339 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/28399e13f59c495d88a8f1990fce1b67 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/28399e13f59c495d88a8f1990fce1b67 2024-11-20T21:16:18,340 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/7d840c5b8eb64bf692b25aec4c32a4b0 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/B/7d840c5b8eb64bf692b25aec4c32a4b0 2024-11-20T21:16:18,342 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/bb1b61f87bfd44cd839a9c0059c7b63c to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/bb1b61f87bfd44cd839a9c0059c7b63c 2024-11-20T21:16:18,342 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/ffa9d5bfb4ad4d1c89fe1f47b3878e9b to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/C/ffa9d5bfb4ad4d1c89fe1f47b3878e9b 2024-11-20T21:16:18,344 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/recovered.edits/551.seqid to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf/recovered.edits/551.seqid 2024-11-20T21:16:18,345 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/6c66f98c982744caf390af289e0ec6cf 2024-11-20T21:16:18,345 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T21:16:18,347 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=143, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:16:18,349 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T21:16:18,350 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T21:16:18,351 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=143, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:16:18,351 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T21:16:18,351 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732137378351"}]},"ts":"9223372036854775807"} 2024-11-20T21:16:18,353 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T21:16:18,353 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6c66f98c982744caf390af289e0ec6cf, NAME => 'TestAcidGuarantees,,1732137353705.6c66f98c982744caf390af289e0ec6cf.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T21:16:18,353 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T21:16:18,353 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732137378353"}]},"ts":"9223372036854775807"} 2024-11-20T21:16:18,355 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T21:16:18,355 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=143, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:16:18,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 28 msec 2024-11-20T21:16:18,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-20T21:16:18,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T21:16:18,443 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:16:18,452 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=249 (was 249), OpenFileDescriptor=447 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=823 (was 809) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=441 (was 998) 2024-11-20T21:16:18,463 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=249, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=823, ProcessCount=11, AvailableMemoryMB=440 2024-11-20T21:16:18,464 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T21:16:18,465 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T21:16:18,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-20T21:16:18,467 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T21:16:18,467 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:18,467 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 144 2024-11-20T21:16:18,468 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T21:16:18,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-20T21:16:18,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742430_1606 (size=1061) 2024-11-20T21:16:18,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-20T21:16:18,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-20T21:16:18,874 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094 2024-11-20T21:16:18,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742431_1607 (size=53) 2024-11-20T21:16:19,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-20T21:16:19,281 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:16:19,281 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2ba04bb2bd029935d159faf0939a1f26, disabling compactions & flushes 2024-11-20T21:16:19,281 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:19,281 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:19,281 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. after waiting 0 ms 2024-11-20T21:16:19,281 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:19,281 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:19,281 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2ba04bb2bd029935d159faf0939a1f26: Waiting for close lock at 1732137379281Disabling compacts and flushes for region at 1732137379281Disabling writes for close at 1732137379281Writing region close event to WAL at 1732137379281Closed at 1732137379281 2024-11-20T21:16:19,282 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T21:16:19,282 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732137379282"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732137379282"}]},"ts":"1732137379282"} 2024-11-20T21:16:19,284 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-20T21:16:19,284 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T21:16:19,284 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732137379284"}]},"ts":"1732137379284"} 2024-11-20T21:16:19,286 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-20T21:16:19,286 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2ba04bb2bd029935d159faf0939a1f26, ASSIGN}] 2024-11-20T21:16:19,287 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2ba04bb2bd029935d159faf0939a1f26, ASSIGN 2024-11-20T21:16:19,287 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2ba04bb2bd029935d159faf0939a1f26, ASSIGN; state=OFFLINE, location=0b16ab3f13b7,36987,1732137236536; forceNewPlan=false, retain=false 2024-11-20T21:16:19,438 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=2ba04bb2bd029935d159faf0939a1f26, regionState=OPENING, regionLocation=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:19,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2ba04bb2bd029935d159faf0939a1f26, ASSIGN because future has completed 2024-11-20T21:16:19,439 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536}] 2024-11-20T21:16:19,593 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:19,593 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7752): Opening region: {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:16:19,594 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:19,594 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:16:19,594 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7794): checking encryption for 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:19,594 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7797): checking classloading for 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:19,595 INFO [StoreOpener-2ba04bb2bd029935d159faf0939a1f26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:19,596 INFO [StoreOpener-2ba04bb2bd029935d159faf0939a1f26-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T21:16:19,596 INFO [StoreOpener-2ba04bb2bd029935d159faf0939a1f26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ba04bb2bd029935d159faf0939a1f26 columnFamilyName A 2024-11-20T21:16:19,596 DEBUG [StoreOpener-2ba04bb2bd029935d159faf0939a1f26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:19,596 INFO [StoreOpener-2ba04bb2bd029935d159faf0939a1f26-1 {}] regionserver.HStore(327): Store=2ba04bb2bd029935d159faf0939a1f26/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:16:19,596 INFO [StoreOpener-2ba04bb2bd029935d159faf0939a1f26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:19,597 INFO [StoreOpener-2ba04bb2bd029935d159faf0939a1f26-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T21:16:19,597 INFO [StoreOpener-2ba04bb2bd029935d159faf0939a1f26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ba04bb2bd029935d159faf0939a1f26 columnFamilyName B 2024-11-20T21:16:19,597 DEBUG [StoreOpener-2ba04bb2bd029935d159faf0939a1f26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:19,598 INFO [StoreOpener-2ba04bb2bd029935d159faf0939a1f26-1 {}] regionserver.HStore(327): Store=2ba04bb2bd029935d159faf0939a1f26/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:16:19,598 INFO [StoreOpener-2ba04bb2bd029935d159faf0939a1f26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:19,599 INFO [StoreOpener-2ba04bb2bd029935d159faf0939a1f26-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T21:16:19,599 INFO [StoreOpener-2ba04bb2bd029935d159faf0939a1f26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ba04bb2bd029935d159faf0939a1f26 columnFamilyName C 2024-11-20T21:16:19,599 DEBUG [StoreOpener-2ba04bb2bd029935d159faf0939a1f26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:19,599 INFO [StoreOpener-2ba04bb2bd029935d159faf0939a1f26-1 {}] regionserver.HStore(327): Store=2ba04bb2bd029935d159faf0939a1f26/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:16:19,599 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:19,599 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1038): replaying wal for 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:19,600 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:19,600 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:19,601 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1048): stopping wal replay for 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:19,601 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1060): Cleaning up temporary data for 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:19,601 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T21:16:19,602 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1093): writing seq id for 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:19,604 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T21:16:19,604 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1114): Opened 2ba04bb2bd029935d159faf0939a1f26; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65506584, jitterRate=-0.023875832557678223}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T21:16:19,604 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:19,605 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1006): Region open journal for 2ba04bb2bd029935d159faf0939a1f26: Running coprocessor pre-open hook at 1732137379594Writing region info on filesystem at 1732137379594Initializing all the Stores at 1732137379594Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732137379594Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732137379595 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732137379595Cleaning up temporary data from old regions at 1732137379601 (+6 ms)Running coprocessor post-open hooks at 1732137379604 (+3 ms)Region opened successfully at 1732137379605 (+1 ms) 2024-11-20T21:16:19,606 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., pid=146, masterSystemTime=1732137379590 2024-11-20T21:16:19,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-20T21:16:19,608 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:19,608 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:19,608 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=2ba04bb2bd029935d159faf0939a1f26, regionState=OPEN, openSeqNum=2, regionLocation=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:19,610 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 because future has completed 2024-11-20T21:16:19,612 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-20T21:16:19,612 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure 2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 in 171 msec 2024-11-20T21:16:19,613 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-20T21:16:19,613 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2ba04bb2bd029935d159faf0939a1f26, ASSIGN in 326 msec 2024-11-20T21:16:19,614 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T21:16:19,614 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732137379614"}]},"ts":"1732137379614"} 2024-11-20T21:16:19,615 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-20T21:16:19,615 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T21:16:19,617 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1500 sec 2024-11-20T21:16:20,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-20T21:16:20,617 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:16:20,618 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43602cdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:16:20,618 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:16:20,618 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:16:20,619 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:16:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:16:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:16:20,619 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1759344e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:16:20,620 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:16:20,620 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:16:20,620 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:20,621 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34578, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:16:20,622 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68b9044b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:16:20,622 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:16:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:16:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:16:20,624 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:16:20,625 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T21:16:20,626 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 0b16ab3f13b7,39687,1732137235812 2024-11-20T21:16:20,626 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5d98a7d7 2024-11-20T21:16:20,626 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T21:16:20,627 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T21:16:20,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-20T21:16:20,629 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T21:16:20,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-20T21:16:20,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742432_1608 (size=1097) 2024-11-20T21:16:21,044 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-20T21:16:21,044 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-20T21:16:21,047 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T21:16:21,049 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2ba04bb2bd029935d159faf0939a1f26, REOPEN/MOVE}] 2024-11-20T21:16:21,049 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2ba04bb2bd029935d159faf0939a1f26, REOPEN/MOVE 2024-11-20T21:16:21,050 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=2ba04bb2bd029935d159faf0939a1f26, regionState=CLOSING, regionLocation=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:21,051 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2ba04bb2bd029935d159faf0939a1f26, REOPEN/MOVE because future has completed 2024-11-20T21:16:21,052 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T21:16:21,052 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536}] 2024-11-20T21:16:21,204 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:21,204 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T21:16:21,204 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing 2ba04bb2bd029935d159faf0939a1f26, disabling compactions & flushes 2024-11-20T21:16:21,204 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:21,204 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:21,204 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. after waiting 0 ms 2024-11-20T21:16:21,204 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:21,208 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T21:16:21,208 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:21,208 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for 2ba04bb2bd029935d159faf0939a1f26: Waiting for close lock at 1732137381204Running coprocessor pre-close hooks at 1732137381204Disabling compacts and flushes for region at 1732137381204Disabling writes for close at 1732137381204Writing region close event to WAL at 1732137381204Running coprocessor post-close hooks at 1732137381208 (+4 ms)Closed at 1732137381208 2024-11-20T21:16:21,208 WARN [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionServer(3299): Not adding moved region record: 2ba04bb2bd029935d159faf0939a1f26 to self. 2024-11-20T21:16:21,210 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:21,211 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=2ba04bb2bd029935d159faf0939a1f26, regionState=CLOSED 2024-11-20T21:16:21,213 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 because future has completed 2024-11-20T21:16:21,215 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-20T21:16:21,215 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure 2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 in 161 msec 2024-11-20T21:16:21,216 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2ba04bb2bd029935d159faf0939a1f26, REOPEN/MOVE; state=CLOSED, location=0b16ab3f13b7,36987,1732137236536; forceNewPlan=false, retain=true 2024-11-20T21:16:21,366 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=2ba04bb2bd029935d159faf0939a1f26, regionState=OPENING, regionLocation=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:21,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2ba04bb2bd029935d159faf0939a1f26, REOPEN/MOVE because future has completed 2024-11-20T21:16:21,368 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536}] 2024-11-20T21:16:21,523 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:21,523 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7752): Opening region: {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} 2024-11-20T21:16:21,524 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:21,524 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T21:16:21,524 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7794): checking encryption for 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:21,524 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7797): checking classloading for 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:21,525 INFO [StoreOpener-2ba04bb2bd029935d159faf0939a1f26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:21,526 INFO [StoreOpener-2ba04bb2bd029935d159faf0939a1f26-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T21:16:21,526 INFO [StoreOpener-2ba04bb2bd029935d159faf0939a1f26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ba04bb2bd029935d159faf0939a1f26 columnFamilyName A 2024-11-20T21:16:21,527 DEBUG [StoreOpener-2ba04bb2bd029935d159faf0939a1f26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:21,528 INFO [StoreOpener-2ba04bb2bd029935d159faf0939a1f26-1 {}] regionserver.HStore(327): Store=2ba04bb2bd029935d159faf0939a1f26/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:16:21,528 INFO [StoreOpener-2ba04bb2bd029935d159faf0939a1f26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:21,528 INFO [StoreOpener-2ba04bb2bd029935d159faf0939a1f26-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T21:16:21,529 INFO [StoreOpener-2ba04bb2bd029935d159faf0939a1f26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ba04bb2bd029935d159faf0939a1f26 columnFamilyName B 2024-11-20T21:16:21,529 DEBUG [StoreOpener-2ba04bb2bd029935d159faf0939a1f26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:21,529 INFO [StoreOpener-2ba04bb2bd029935d159faf0939a1f26-1 {}] regionserver.HStore(327): Store=2ba04bb2bd029935d159faf0939a1f26/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:16:21,529 INFO [StoreOpener-2ba04bb2bd029935d159faf0939a1f26-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:21,530 INFO [StoreOpener-2ba04bb2bd029935d159faf0939a1f26-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-20T21:16:21,530 INFO [StoreOpener-2ba04bb2bd029935d159faf0939a1f26-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2ba04bb2bd029935d159faf0939a1f26 columnFamilyName C 2024-11-20T21:16:21,530 DEBUG [StoreOpener-2ba04bb2bd029935d159faf0939a1f26-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:21,530 INFO [StoreOpener-2ba04bb2bd029935d159faf0939a1f26-1 {}] regionserver.HStore(327): Store=2ba04bb2bd029935d159faf0939a1f26/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T21:16:21,530 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:21,530 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1038): replaying wal for 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:21,531 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:21,532 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:21,533 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1048): stopping wal replay for 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:21,533 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1060): Cleaning up temporary data for 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:21,533 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-20T21:16:21,534 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1093): writing seq id for 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:21,535 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1114): Opened 2ba04bb2bd029935d159faf0939a1f26; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64704300, jitterRate=-0.035830795764923096}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-20T21:16:21,535 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:21,536 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1006): Region open journal for 2ba04bb2bd029935d159faf0939a1f26: Running coprocessor pre-open hook at 1732137381524Writing region info on filesystem at 1732137381524Initializing all the Stores at 1732137381525 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732137381525Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732137381525Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732137381525Cleaning up temporary data from old regions at 1732137381533 (+8 ms)Running coprocessor post-open hooks at 1732137381535 (+2 ms)Region opened successfully at 1732137381536 (+1 ms) 2024-11-20T21:16:21,537 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., pid=151, masterSystemTime=1732137381520 2024-11-20T21:16:21,538 DEBUG [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:21,538 INFO [RS_OPEN_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:21,539 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=2ba04bb2bd029935d159faf0939a1f26, regionState=OPEN, openSeqNum=5, regionLocation=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:21,540 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 because future has completed 2024-11-20T21:16:21,542 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=149 2024-11-20T21:16:21,542 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure 2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 in 172 msec 2024-11-20T21:16:21,544 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-20T21:16:21,544 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2ba04bb2bd029935d159faf0939a1f26, REOPEN/MOVE in 493 msec 2024-11-20T21:16:21,545 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-20T21:16:21,545 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 497 msec 2024-11-20T21:16:21,547 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 917 msec 2024-11-20T21:16:21,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-20T21:16:21,548 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:16:21,548 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e76e6cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:16:21,549 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:16:21,549 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:16:21,549 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:16:21,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:16:21,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:16:21,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fbbe26f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:16:21,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:16:21,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:16:21,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:21,551 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34600, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:16:21,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@156d1106, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:16:21,551 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11971b93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:16:21,551 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:16:21,552 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:16:21,552 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:16:21,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:16:21,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:16:21,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@179dedd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:16:21,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:16:21,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:16:21,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:21,553 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:16:21,553 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1459e840, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:16:21,554 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24560e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:16:21,554 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:16:21,554 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:16:21,555 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:16:21,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:16:21,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:16:21,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b9b8424, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:16:21,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:16:21,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:16:21,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:21,556 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34626, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:16:21,556 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e4f4f33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:16:21,557 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e83be8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:16:21,557 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:16:21,557 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:16:21,557 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:16:21,557 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:16:21,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:16:21,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15a63b90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:16:21,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:16:21,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:16:21,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:21,558 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34646, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:16:21,559 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3433b99e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:16:21,559 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49c8d9ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:16:21,559 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:16:21,560 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:16:21,560 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:16:21,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:16:21,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:16:21,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f539e54, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:16:21,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:16:21,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:16:21,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:21,561 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:16:21,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b58a75d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:16:21,562 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cb57557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:16:21,562 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:16:21,562 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:16:21,562 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:16:21,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:16:21,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:16:21,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@529ac1c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:16:21,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:16:21,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:16:21,563 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:21,563 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:16:21,564 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3efd2f8b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:16:21,564 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e0b6ba1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:16:21,564 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:16:21,564 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:16:21,565 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:16:21,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:16:21,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:16:21,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5822c562, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:16:21,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:16:21,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:16:21,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:21,566 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:16:21,567 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2efbf915, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:16:21,567 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19bc8537, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:16:21,567 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:16:21,567 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:16:21,568 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:16:21,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:16:21,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:16:21,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@382200, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:16:21,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:16:21,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:16:21,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:21,569 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:16:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67cc9abb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:16:21,570 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7595d14b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:16:21,570 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:16:21,570 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:16:21,571 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:16:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:16:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:16:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@485d4fce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:16:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:16:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:16:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:21,572 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34744, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:16:21,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54e456ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:16:21,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bf79e45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:16:21,573 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 0b16ab3f13b7,39687,-1 for getting cluster id 2024-11-20T21:16:21,573 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T21:16:21,573 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '2a21c2b7-ffb9-4430-8151-95213010e3ec' 2024-11-20T21:16:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T21:16:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "2a21c2b7-ffb9-4430-8151-95213010e3ec" 2024-11-20T21:16:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35ed86dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:16:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [0b16ab3f13b7,39687,-1] 2024-11-20T21:16:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T21:16:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:21,575 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T21:16:21,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ce6d548, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T21:16:21,576 DEBUG [Thread-2692 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:16:21,576 DEBUG [Thread-2673 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:16:21,576 DEBUG [Thread-2688 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:16:21,576 DEBUG [Thread-2684 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:16:21,576 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:16:21,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:16:21,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:16:21,576 DEBUG [Thread-2686 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:16:21,577 DEBUG [Thread-2677 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:16:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:16:21,577 DEBUG [Thread-2679 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:16:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:16:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:16:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:16:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:16:21,577 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:16:21,578 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:16:21,578 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:16:21,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:16:21,578 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:16:21,578 DEBUG [Thread-2675 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:16:21,578 DEBUG [Thread-2677 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:16:21,578 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:16:21,578 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:16:21,578 DEBUG [Thread-2677 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:16:21,579 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44250, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:16:21,579 DEBUG [Thread-2675 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:16:21,579 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44280, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:16:21,579 DEBUG [Thread-2675 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:16:21,579 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:16:21,579 DEBUG [Thread-2681 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:16:21,580 DEBUG [Thread-2690 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T21:16:21,580 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44264, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:16:21,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5] 2024-11-20T21:16:21,580 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:16:21,580 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5] 2024-11-20T21:16:21,580 DEBUG [Thread-2681 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:16:21,581 DEBUG [Thread-2681 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:16:21,581 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:16:21,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=0b16ab3f13b7,36987,1732137236536, seqNum=-1] 2024-11-20T21:16:21,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T21:16:21,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5] 2024-11-20T21:16:21,581 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5] 2024-11-20T21:16:21,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5] 2024-11-20T21:16:21,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44332, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:16:21,582 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T21:16:21,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-20T21:16:21,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5] 2024-11-20T21:16:21,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T21:16:21,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5] 2024-11-20T21:16:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5] 2024-11-20T21:16:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5] 2024-11-20T21:16:21,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5] 2024-11-20T21:16:21,585 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:16:21,587 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:16:21,587 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:16:21,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:21,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2ba04bb2bd029935d159faf0939a1f26 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T21:16:21,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=A 2024-11-20T21:16:21,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:21,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=B 2024-11-20T21:16:21,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:21,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=C 2024-11-20T21:16:21,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:21,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c32b1676b0cf4902ad2a54132b794ed8_2ba04bb2bd029935d159faf0939a1f26 is 50, key is test_row_0/A:col10/1732137381586/Put/seqid=0 2024-11-20T21:16:21,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:21,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137441615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:21,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732137381599, max timestamp=1732137381617Num uniques 859; 2024-11-20T21:16:21,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T21:16:21,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732137381610, max timestamp=1732137381617Num uniques 859; 2024-11-20T21:16:21,626 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-20T21:16:21,627 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:21,627 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:21,627 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:21,627 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732137381610, max timestamp=1732137381617Num uniques 859; 2024-11-20T21:16:21,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-20T21:16:21,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742433_1609 (size=12154) 2024-11-20T21:16:21,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:21,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137441652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:21,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:21,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:21,658 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:21,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:21,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44310 deadline: 1732137441655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:21,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:21,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44216 deadline: 1732137441657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:21,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:21,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:21,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137441655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:21,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:21,669 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:21,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:21,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:21,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:21,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T21:16:21,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137441737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:21,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-20T21:16:21,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:21,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:21,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:21,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:21,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:21,742 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:21,742 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:21,742 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:21,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:21,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:21,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44216 deadline: 1732137441768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:21,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:21,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137441769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:21,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:21,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:21,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:21,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:21,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:21,777 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:21,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137441781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:21,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44310 deadline: 1732137441781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:21,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:21,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:21,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:21,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:21,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:21,783 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:21,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-20T21:16:21,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T21:16:21,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:21,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:21,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:21,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:21,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:21,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:21,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:21,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137441948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:21,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:21,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:21,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:21,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44216 deadline: 1732137441977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:21,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:21,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:21,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:21,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137441987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:21,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137441987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:21,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:21,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:21,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:21,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:21,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:21,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44310 deadline: 1732137441989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:21,993 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:21,993 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:21,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:21,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:21,994 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:22,045 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:22,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-20T21:16:22,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:22,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:22,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:22,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:22,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:22,063 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120c32b1676b0cf4902ad2a54132b794ed8_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c32b1676b0cf4902ad2a54132b794ed8_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:22,073 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/dfe4d180a85d4779a6f70ca1852f2a0b, store: [table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:22,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/dfe4d180a85d4779a6f70ca1852f2a0b is 175, key is test_row_0/A:col10/1732137381586/Put/seqid=0 2024-11-20T21:16:22,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742434_1610 (size=30955) 2024-11-20T21:16:22,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T21:16:22,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-20T21:16:22,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:22,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:22,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:22,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:22,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:22,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:22,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:22,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137442257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:22,260 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:22,260 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:22,260 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:22,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:22,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44216 deadline: 1732137442287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:22,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:22,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:22,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:22,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137442296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:22,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:22,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137442296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:22,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:22,301 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:22,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:22,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44310 deadline: 1732137442296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:22,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:22,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:22,301 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:22,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:22,301 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:22,302 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:22,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-20T21:16:22,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:22,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:22,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:22,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:22,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:22,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:22,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-20T21:16:22,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:22,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:22,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:22,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:22,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:22,525 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/dfe4d180a85d4779a6f70ca1852f2a0b 2024-11-20T21:16:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:22,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/a9382244120a48c89147f8a3df6aea08 is 50, key is test_row_0/B:col10/1732137381586/Put/seqid=0 2024-11-20T21:16:22,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742435_1611 (size=12001) 2024-11-20T21:16:22,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-20T21:16:22,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:22,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:22,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:22,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:22,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T21:16:22,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:22,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137442767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:22,770 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:22,770 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:22,770 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:22,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:22,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44216 deadline: 1732137442797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:22,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:22,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:22,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:22,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137442807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:22,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:22,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:22,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:22,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137442807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:22,809 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:22,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:22,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:22,809 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:22,819 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:22,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44310 deadline: 1732137442816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:22,819 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:22,819 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:22,819 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:22,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-20T21:16:22,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:22,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:22,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:22,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:22,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:22,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:22,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/a9382244120a48c89147f8a3df6aea08 2024-11-20T21:16:22,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-20T21:16:22,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:22,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:22,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:22,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:22,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:22,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:22,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/2f9f320357084913be1183ad15019782 is 50, key is test_row_0/C:col10/1732137381586/Put/seqid=0 2024-11-20T21:16:22,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742436_1612 (size=12001) 2024-11-20T21:16:22,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/2f9f320357084913be1183ad15019782 2024-11-20T21:16:23,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/dfe4d180a85d4779a6f70ca1852f2a0b as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/dfe4d180a85d4779a6f70ca1852f2a0b 2024-11-20T21:16:23,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/dfe4d180a85d4779a6f70ca1852f2a0b, entries=150, sequenceid=14, filesize=30.2 K 2024-11-20T21:16:23,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/a9382244120a48c89147f8a3df6aea08 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/a9382244120a48c89147f8a3df6aea08 2024-11-20T21:16:23,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/a9382244120a48c89147f8a3df6aea08, entries=150, sequenceid=14, filesize=11.7 K 2024-11-20T21:16:23,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/2f9f320357084913be1183ad15019782 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/2f9f320357084913be1183ad15019782 2024-11-20T21:16:23,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/2f9f320357084913be1183ad15019782, entries=150, sequenceid=14, filesize=11.7 K 2024-11-20T21:16:23,073 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=137.86 KB/141172 for 2ba04bb2bd029935d159faf0939a1f26 in 1482ms, sequenceid=14, compaction requested=false 2024-11-20T21:16:23,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:23,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-20T21:16:23,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:23,140 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 2ba04bb2bd029935d159faf0939a1f26 3/3 column families, dataSize=137.86 KB heapSize=393.31 KB 2024-11-20T21:16:23,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=A 2024-11-20T21:16:23,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:16:23,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=B 2024-11-20T21:16:23,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:16:23,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=C 2024-11-20T21:16:23,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:16:23,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b5c7f043673941f99f892466c4fb467b_2ba04bb2bd029935d159faf0939a1f26 is 50, key is test_row_0/A:col10/1732137381644/Put/seqid=0 2024-11-20T21:16:23,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742437_1613 (size=12154) 2024-11-20T21:16:23,536 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T21:16:23,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,588 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b5c7f043673941f99f892466c4fb467b_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b5c7f043673941f99f892466c4fb467b_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:23,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/c8e889f9273c4c068a701c9da553354b, store: [table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:23,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/c8e889f9273c4c068a701c9da553354b is 175, key is test_row_0/A:col10/1732137381644/Put/seqid=0 2024-11-20T21:16:23,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742438_1614 (size=30955) 2024-11-20T21:16:23,615 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/c8e889f9273c4c068a701c9da553354b 2024-11-20T21:16:23,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/89fec769e2be4b538aa90fcac6ff53b3 is 50, key is test_row_0/B:col10/1732137381644/Put/seqid=0 2024-11-20T21:16:23,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742439_1615 (size=12001) 2024-11-20T21:16:23,666 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/89fec769e2be4b538aa90fcac6ff53b3 2024-11-20T21:16:23,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/fc0e3fbf391f4820881047653c9e38aa is 50, key is test_row_0/C:col10/1732137381644/Put/seqid=0 2024-11-20T21:16:23,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742440_1616 (size=12001) 2024-11-20T21:16:23,691 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/fc0e3fbf391f4820881047653c9e38aa 2024-11-20T21:16:23,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/c8e889f9273c4c068a701c9da553354b as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/c8e889f9273c4c068a701c9da553354b 2024-11-20T21:16:23,702 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/c8e889f9273c4c068a701c9da553354b, entries=150, sequenceid=51, filesize=30.2 K 2024-11-20T21:16:23,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/89fec769e2be4b538aa90fcac6ff53b3 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/89fec769e2be4b538aa90fcac6ff53b3 2024-11-20T21:16:23,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,706 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/89fec769e2be4b538aa90fcac6ff53b3, entries=150, sequenceid=51, filesize=11.7 K 2024-11-20T21:16:23,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/fc0e3fbf391f4820881047653c9e38aa as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/fc0e3fbf391f4820881047653c9e38aa 2024-11-20T21:16:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,710 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/fc0e3fbf391f4820881047653c9e38aa, entries=150, sequenceid=51, filesize=11.7 K 2024-11-20T21:16:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,711 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~137.86 KB/141172, heapSize ~393.27 KB/402704, currentSize=0 B/0 for 2ba04bb2bd029935d159faf0939a1f26 in 571ms, sequenceid=51, compaction requested=false 2024-11-20T21:16:23,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:23,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-20T21:16:23,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-20T21:16:23,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,717 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-20T21:16:23,717 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1280 sec 2024-11-20T21:16:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,719 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.1370 sec 2024-11-20T21:16:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-20T21:16:23,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,727 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:16:23,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,727 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:16:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-20T21:16:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,729 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:16:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-20T21:16:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,735 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:16:23,735 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:16:23,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-20T21:16:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:23,851 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2ba04bb2bd029935d159faf0939a1f26 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T21:16:23,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=A 2024-11-20T21:16:23,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:23,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=B 2024-11-20T21:16:23,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:23,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=C 2024-11-20T21:16:23,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:23,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:23,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209f0bc6962bb3480299421675f0f381c9_2ba04bb2bd029935d159faf0939a1f26 is 50, key is test_row_0/A:col10/1732137383845/Put/seqid=0 2024-11-20T21:16:23,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732137383863, max timestamp=1732137383876Num uniques 859; 2024-11-20T21:16:23,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732137383863, max timestamp=1732137383876Num uniques 859; 2024-11-20T21:16:23,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-20T21:16:23,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-20T21:16:23,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1732137383863, max timestamp=1732137383876Num uniques 859; 2024-11-20T21:16:23,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-20T21:16:23,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-20T21:16:23,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:23,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742441_1617 (size=21918) 2024-11-20T21:16:23,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:23,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:23,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:23,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:23,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:23,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137443906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:23,916 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:23,916 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:23,916 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:23,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:23,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137443915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:23,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:23,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44310 deadline: 1732137443915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:23,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:23,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:23,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44216 deadline: 1732137443916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:23,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:23,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137443918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:23,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:23,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:23,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:23,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:23,924 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:23,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:23,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:23,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:23,925 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:24,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137444027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:24,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137444027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:24,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:24,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44310 deadline: 1732137444027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:24,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137444028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:24,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:24,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44216 deadline: 1732137444028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:24,033 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:24,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:24,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:24,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:24,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-20T21:16:24,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:24,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:24,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:24,042 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:24,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:24,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:24,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-20T21:16:24,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-20T21:16:24,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:24,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:24,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:24,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:24,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:24,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:24,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:24,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137444237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:24,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:24,242 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:24,242 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:24,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137444237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:24,242 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:24,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:24,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44310 deadline: 1732137444237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:24,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:24,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:24,242 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:24,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:24,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137444238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:24,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:24,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:24,243 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:24,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:24,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44216 deadline: 1732137444238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:24,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:24,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:24,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:24,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:24,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:24,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:24,289 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:24,292 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411209f0bc6962bb3480299421675f0f381c9_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209f0bc6962bb3480299421675f0f381c9_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:24,293 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/9015cc75a4e342129b755478c3186bfd, store: [table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:24,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/9015cc75a4e342129b755478c3186bfd is 175, key is test_row_0/A:col10/1732137383845/Put/seqid=0 2024-11-20T21:16:24,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742442_1618 (size=65323) 2024-11-20T21:16:24,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-20T21:16:24,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:24,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-20T21:16:24,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:24,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:24,357 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:24,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:24,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:24,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-20T21:16:24,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:24,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:24,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:24,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:24,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:24,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:24,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137444546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:24,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:24,552 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:24,552 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:24,552 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137444547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:24,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:24,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:24,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:24,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137444548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:24,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:24,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:24,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:24,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:24,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44310 deadline: 1732137444548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:24,553 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:24,553 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:24,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:24,553 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:24,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44216 deadline: 1732137444548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:24,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:24,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:24,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:24,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-20T21:16:24,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:24,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:24,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:24,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:24,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:24,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:24,703 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/9015cc75a4e342129b755478c3186bfd 2024-11-20T21:16:24,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/9ac3e080379241deab1f7b3ec923d256 is 50, key is test_row_0/B:col10/1732137383845/Put/seqid=0 2024-11-20T21:16:24,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742443_1619 (size=12001) 2024-11-20T21:16:24,816 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-20T21:16:24,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-20T21:16:24,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:24,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:24,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:24,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:24,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:24,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:24,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-20T21:16:24,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-20T21:16:24,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:24,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:24,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:24,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:24,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:24,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:25,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:25,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137445056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:25,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:25,061 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:25,061 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:25,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137445058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:25,061 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:25,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:25,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:25,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:25,061 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:25,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137445058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:25,062 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:25,062 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:25,062 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:25,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:25,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44310 deadline: 1732137445067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:25,070 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:25,070 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:25,070 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:25,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:25,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44216 deadline: 1732137445067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:25,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/9ac3e080379241deab1f7b3ec923d256 2024-11-20T21:16:25,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/e51ade5eb9c64bcfbd0c6dd068d5b8f8 is 50, key is test_row_0/C:col10/1732137383845/Put/seqid=0 2024-11-20T21:16:25,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742444_1620 (size=12001) 2024-11-20T21:16:25,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-20T21:16:25,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:25,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:25,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:25,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:25,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:25,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:25,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-20T21:16:25,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:25,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:25,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:25,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:25,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:25,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:25,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-20T21:16:25,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:25,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:25,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:25,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:25,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:25,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:25,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/e51ade5eb9c64bcfbd0c6dd068d5b8f8 2024-11-20T21:16:25,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/9015cc75a4e342129b755478c3186bfd as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/9015cc75a4e342129b755478c3186bfd 2024-11-20T21:16:25,571 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/9015cc75a4e342129b755478c3186bfd, entries=350, sequenceid=61, filesize=63.8 K 2024-11-20T21:16:25,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/9ac3e080379241deab1f7b3ec923d256 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/9ac3e080379241deab1f7b3ec923d256 2024-11-20T21:16:25,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/9ac3e080379241deab1f7b3ec923d256, entries=150, sequenceid=61, filesize=11.7 K 2024-11-20T21:16:25,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/e51ade5eb9c64bcfbd0c6dd068d5b8f8 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/e51ade5eb9c64bcfbd0c6dd068d5b8f8 2024-11-20T21:16:25,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/e51ade5eb9c64bcfbd0c6dd068d5b8f8, entries=150, sequenceid=61, filesize=11.7 K 2024-11-20T21:16:25,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for 2ba04bb2bd029935d159faf0939a1f26 in 1732ms, sequenceid=61, compaction requested=true 2024-11-20T21:16:25,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:25,583 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:16:25,585 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:16:25,585 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 2ba04bb2bd029935d159faf0939a1f26/A is initiating minor compaction (all files) 2024-11-20T21:16:25,585 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ba04bb2bd029935d159faf0939a1f26/A in TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:25,585 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/dfe4d180a85d4779a6f70ca1852f2a0b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/c8e889f9273c4c068a701c9da553354b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/9015cc75a4e342129b755478c3186bfd] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp, totalSize=124.3 K 2024-11-20T21:16:25,585 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:25,586 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. files: [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/dfe4d180a85d4779a6f70ca1852f2a0b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/c8e889f9273c4c068a701c9da553354b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/9015cc75a4e342129b755478c3186bfd] 2024-11-20T21:16:25,586 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfe4d180a85d4779a6f70ca1852f2a0b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732137381586 2024-11-20T21:16:25,586 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8e889f9273c4c068a701c9da553354b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732137381644 2024-11-20T21:16:25,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ba04bb2bd029935d159faf0939a1f26:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:16:25,587 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9015cc75a4e342129b755478c3186bfd, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732137383809 2024-11-20T21:16:25,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:25,587 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:16:25,589 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:16:25,589 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 2ba04bb2bd029935d159faf0939a1f26/B is initiating minor compaction (all files) 2024-11-20T21:16:25,589 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ba04bb2bd029935d159faf0939a1f26/B in TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:25,589 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/a9382244120a48c89147f8a3df6aea08, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/89fec769e2be4b538aa90fcac6ff53b3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/9ac3e080379241deab1f7b3ec923d256] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp, totalSize=35.2 K 2024-11-20T21:16:25,589 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting a9382244120a48c89147f8a3df6aea08, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732137381586 2024-11-20T21:16:25,590 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 89fec769e2be4b538aa90fcac6ff53b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732137381644 2024-11-20T21:16:25,591 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ac3e080379241deab1f7b3ec923d256, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732137383845 2024-11-20T21:16:25,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ba04bb2bd029935d159faf0939a1f26:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:16:25,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:25,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ba04bb2bd029935d159faf0939a1f26:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:16:25,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:16:25,597 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:25,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-20T21:16:25,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:25,599 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 2ba04bb2bd029935d159faf0939a1f26 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-20T21:16:25,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=A 2024-11-20T21:16:25,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:16:25,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=B 2024-11-20T21:16:25,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:16:25,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=C 2024-11-20T21:16:25,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:16:25,600 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112046a8ddb6a58b45f6a0f74fa4579df2b3_2ba04bb2bd029935d159faf0939a1f26 store=[table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:25,603 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112046a8ddb6a58b45f6a0f74fa4579df2b3_2ba04bb2bd029935d159faf0939a1f26, store=[table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:25,603 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112046a8ddb6a58b45f6a0f74fa4579df2b3_2ba04bb2bd029935d159faf0939a1f26 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:25,614 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ba04bb2bd029935d159faf0939a1f26#B#compaction#531 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:25,615 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/3ab14844bcc24834ac7ae016f5b3b17e is 50, key is test_row_0/B:col10/1732137383845/Put/seqid=0 2024-11-20T21:16:25,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b7988188bfcd4a3ab4fa3120fd1aac6a_2ba04bb2bd029935d159faf0939a1f26 is 50, key is test_row_0/A:col10/1732137383915/Put/seqid=0 2024-11-20T21:16:25,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742445_1621 (size=4469) 2024-11-20T21:16:25,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742446_1622 (size=12104) 2024-11-20T21:16:25,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742447_1623 (size=12154) 2024-11-20T21:16:25,698 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/3ab14844bcc24834ac7ae016f5b3b17e as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/3ab14844bcc24834ac7ae016f5b3b17e 2024-11-20T21:16:25,707 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2ba04bb2bd029935d159faf0939a1f26/B of 2ba04bb2bd029935d159faf0939a1f26 into 3ab14844bcc24834ac7ae016f5b3b17e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:25,707 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:25,707 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., storeName=2ba04bb2bd029935d159faf0939a1f26/B, priority=13, startTime=1732137385587; duration=0sec 2024-11-20T21:16:25,707 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:16:25,707 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ba04bb2bd029935d159faf0939a1f26:B 2024-11-20T21:16:25,707 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:16:25,708 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:16:25,708 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 2ba04bb2bd029935d159faf0939a1f26/C is initiating minor compaction (all files) 2024-11-20T21:16:25,708 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ba04bb2bd029935d159faf0939a1f26/C in TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:25,708 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/2f9f320357084913be1183ad15019782, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/fc0e3fbf391f4820881047653c9e38aa, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/e51ade5eb9c64bcfbd0c6dd068d5b8f8] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp, totalSize=35.2 K 2024-11-20T21:16:25,708 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f9f320357084913be1183ad15019782, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732137381586 2024-11-20T21:16:25,709 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting fc0e3fbf391f4820881047653c9e38aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732137381644 2024-11-20T21:16:25,709 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting e51ade5eb9c64bcfbd0c6dd068d5b8f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732137383845 2024-11-20T21:16:25,714 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ba04bb2bd029935d159faf0939a1f26#C#compaction#533 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:25,715 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/22a9705afcb046f082e1af32447ce711 is 50, key is test_row_0/C:col10/1732137383845/Put/seqid=0 2024-11-20T21:16:25,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742448_1624 (size=12104) 2024-11-20T21:16:25,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-20T21:16:26,058 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ba04bb2bd029935d159faf0939a1f26#A#compaction#530 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:26,058 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/2cfcdbed74e84ffdb9899c50f71615cc is 175, key is test_row_0/A:col10/1732137383845/Put/seqid=0 2024-11-20T21:16:26,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742449_1625 (size=31058) 2024-11-20T21:16:26,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:26,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:26,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:26,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44216 deadline: 1732137446077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:26,083 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 0b16ab3f13b7,36987,1732137236536 for 'test_row_0' in 2ba04bb2bd029935d159faf0939a1f26 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T21:16:26,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:26,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:26,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:26,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137446081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:26,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:26,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:26,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137446083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:26,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:26,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137446083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:26,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44310 deadline: 1732137446087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:26,089 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:26,089 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:26,089 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:26,089 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:26,089 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:26,089 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:26,089 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 0b16ab3f13b7,36987,1732137236536 for 'test_row_2' in 2ba04bb2bd029935d159faf0939a1f26 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T21:16:26,090 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:26,090 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:26,090 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:26,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,102 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120b7988188bfcd4a3ab4fa3120fd1aac6a_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b7988188bfcd4a3ab4fa3120fd1aac6a_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:26,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/e675376333e14aa49750696d0113221c, store: [table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:26,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/e675376333e14aa49750696d0113221c is 175, key is test_row_0/A:col10/1732137383915/Put/seqid=0 2024-11-20T21:16:26,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742450_1626 (size=30955) 2024-11-20T21:16:26,109 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/e675376333e14aa49750696d0113221c 2024-11-20T21:16:26,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/289b7f3a3bf14dd09b1bc5e1006a92ef is 50, key is test_row_0/B:col10/1732137383915/Put/seqid=0 2024-11-20T21:16:26,126 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/22a9705afcb046f082e1af32447ce711 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/22a9705afcb046f082e1af32447ce711 2024-11-20T21:16:26,134 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2ba04bb2bd029935d159faf0939a1f26/C of 2ba04bb2bd029935d159faf0939a1f26 into 22a9705afcb046f082e1af32447ce711(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:26,134 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:26,134 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., storeName=2ba04bb2bd029935d159faf0939a1f26/C, priority=13, startTime=1732137385591; duration=0sec 2024-11-20T21:16:26,134 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:26,134 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ba04bb2bd029935d159faf0939a1f26:C 2024-11-20T21:16:26,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742451_1627 (size=12001) 2024-11-20T21:16:26,150 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/289b7f3a3bf14dd09b1bc5e1006a92ef 2024-11-20T21:16:26,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/384107f1e76f4f6890e75031d752d6e2 is 50, key is test_row_0/C:col10/1732137383915/Put/seqid=0 2024-11-20T21:16:26,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742452_1628 (size=12001) 2024-11-20T21:16:26,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137446197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:26,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:26,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:26,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:26,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137446199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:26,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137446199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:26,203 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:26,204 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:26,204 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:26,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:26,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:26,204 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:26,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137446407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:26,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:26,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:26,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137446407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:26,412 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:26,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137446407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:26,412 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:26,413 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:26,413 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:26,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:26,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:26,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:26,469 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/2cfcdbed74e84ffdb9899c50f71615cc as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/2cfcdbed74e84ffdb9899c50f71615cc 2024-11-20T21:16:26,472 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2ba04bb2bd029935d159faf0939a1f26/A of 2ba04bb2bd029935d159faf0939a1f26 into 2cfcdbed74e84ffdb9899c50f71615cc(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:26,472 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:26,472 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., storeName=2ba04bb2bd029935d159faf0939a1f26/A, priority=13, startTime=1732137385583; duration=0sec 2024-11-20T21:16:26,472 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:26,472 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ba04bb2bd029935d159faf0939a1f26:A 2024-11-20T21:16:26,593 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/384107f1e76f4f6890e75031d752d6e2 2024-11-20T21:16:26,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/e675376333e14aa49750696d0113221c as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/e675376333e14aa49750696d0113221c 2024-11-20T21:16:26,605 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/e675376333e14aa49750696d0113221c, entries=150, sequenceid=97, filesize=30.2 K 2024-11-20T21:16:26,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/289b7f3a3bf14dd09b1bc5e1006a92ef as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/289b7f3a3bf14dd09b1bc5e1006a92ef 2024-11-20T21:16:26,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,610 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/289b7f3a3bf14dd09b1bc5e1006a92ef, entries=150, sequenceid=97, filesize=11.7 K 2024-11-20T21:16:26,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/384107f1e76f4f6890e75031d752d6e2 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/384107f1e76f4f6890e75031d752d6e2 2024-11-20T21:16:26,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,625 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/384107f1e76f4f6890e75031d752d6e2, entries=150, sequenceid=97, filesize=11.7 K 2024-11-20T21:16:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,629 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for 2ba04bb2bd029935d159faf0939a1f26 in 1030ms, sequenceid=97, compaction requested=false 2024-11-20T21:16:26,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:26,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:26,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-20T21:16:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-20T21:16:26,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,633 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-20T21:16:26,633 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8960 sec 2024-11-20T21:16:26,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.9070 sec 2024-11-20T21:16:26,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,725 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2ba04bb2bd029935d159faf0939a1f26 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-20T21:16:26,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=A 2024-11-20T21:16:26,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:26,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=B 2024-11-20T21:16:26,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:26,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=C 2024-11-20T21:16:26,726 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:26,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:26,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206922d61236a44508b39d880593e4167c_2ba04bb2bd029935d159faf0939a1f26 is 50, key is test_row_1/A:col10/1732137386081/Put/seqid=0 2024-11-20T21:16:26,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:26,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742453_1629 (size=12154) 2024-11-20T21:16:26,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732137386779, max timestamp=1732137386791Num uniques 859; 2024-11-20T21:16:26,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732137386779, max timestamp=1732137386791Num uniques 859; 2024-11-20T21:16:26,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T21:16:26,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T21:16:26,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1732137386779, max timestamp=1732137386791Num uniques 859; 2024-11-20T21:16:26,795 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-20T21:16:26,884 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732137386867, max timestamp=1732137386879Num uniques 1018; 2024-11-20T21:16:26,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-20T21:16:26,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732137386867, max timestamp=1732137386879Num uniques 1018; 2024-11-20T21:16:26,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-20T21:16:26,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1732137386867, max timestamp=1732137386879Num uniques 1018; 2024-11-20T21:16:26,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-20T21:16:26,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:26,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137446912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:26,917 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:26,917 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:26,917 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:26,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:26,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137446916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:26,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:26,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:26,923 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:26,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:26,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137446917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:26,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:26,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:26,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:27,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137447026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:27,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137447026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:27,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:27,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:27,029 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:27,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:27,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137447027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:27,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:27,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:27,029 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:27,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:27,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:27,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:27,182 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:27,185 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411206922d61236a44508b39d880593e4167c_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206922d61236a44508b39d880593e4167c_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:27,186 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/e61a48eadc8f4e85be992a048b132481, store: [table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:27,187 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/e61a48eadc8f4e85be992a048b132481 is 175, key is test_row_1/A:col10/1732137386081/Put/seqid=0 2024-11-20T21:16:27,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742454_1630 (size=30951) 2024-11-20T21:16:27,195 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=110, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/e61a48eadc8f4e85be992a048b132481 2024-11-20T21:16:27,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/91e70128629646be9d03e4783b38c988 is 50, key is test_row_1/B:col10/1732137386081/Put/seqid=0 2024-11-20T21:16:27,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742455_1631 (size=9657) 2024-11-20T21:16:27,217 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/91e70128629646be9d03e4783b38c988 2024-11-20T21:16:27,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/beb960b23b4642709eec1ae58d75403a is 50, key is test_row_1/C:col10/1732137386081/Put/seqid=0 2024-11-20T21:16:27,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742456_1632 (size=9657) 2024-11-20T21:16:27,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/beb960b23b4642709eec1ae58d75403a 2024-11-20T21:16:27,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/e61a48eadc8f4e85be992a048b132481 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/e61a48eadc8f4e85be992a048b132481 2024-11-20T21:16:27,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137447238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:27,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:27,239 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:27,239 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:27,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137447238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:27,239 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:27,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:27,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137447238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:27,239 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:27,239 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:27,239 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:27,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:27,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:27,240 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:27,241 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/e61a48eadc8f4e85be992a048b132481, entries=150, sequenceid=110, filesize=30.2 K 2024-11-20T21:16:27,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/91e70128629646be9d03e4783b38c988 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/91e70128629646be9d03e4783b38c988 2024-11-20T21:16:27,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/91e70128629646be9d03e4783b38c988, entries=100, sequenceid=110, filesize=9.4 K 2024-11-20T21:16:27,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/beb960b23b4642709eec1ae58d75403a as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/beb960b23b4642709eec1ae58d75403a 2024-11-20T21:16:27,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/beb960b23b4642709eec1ae58d75403a, entries=100, sequenceid=110, filesize=9.4 K 2024-11-20T21:16:27,253 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 2ba04bb2bd029935d159faf0939a1f26 in 528ms, sequenceid=110, compaction requested=true 2024-11-20T21:16:27,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:27,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ba04bb2bd029935d159faf0939a1f26:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:16:27,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:27,253 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:16:27,253 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:16:27,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ba04bb2bd029935d159faf0939a1f26:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:16:27,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:27,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ba04bb2bd029935d159faf0939a1f26:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:16:27,253 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:16:27,259 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92964 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:16:27,259 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:16:27,259 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 2ba04bb2bd029935d159faf0939a1f26/A is initiating minor compaction (all files) 2024-11-20T21:16:27,259 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 2ba04bb2bd029935d159faf0939a1f26/B is initiating minor compaction (all files) 2024-11-20T21:16:27,259 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ba04bb2bd029935d159faf0939a1f26/A in TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:27,259 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ba04bb2bd029935d159faf0939a1f26/B in TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:27,259 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/2cfcdbed74e84ffdb9899c50f71615cc, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/e675376333e14aa49750696d0113221c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/e61a48eadc8f4e85be992a048b132481] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp, totalSize=90.8 K 2024-11-20T21:16:27,259 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:27,259 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. files: [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/2cfcdbed74e84ffdb9899c50f71615cc, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/e675376333e14aa49750696d0113221c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/e61a48eadc8f4e85be992a048b132481] 2024-11-20T21:16:27,259 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/3ab14844bcc24834ac7ae016f5b3b17e, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/289b7f3a3bf14dd09b1bc5e1006a92ef, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/91e70128629646be9d03e4783b38c988] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp, totalSize=33.0 K 2024-11-20T21:16:27,260 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cfcdbed74e84ffdb9899c50f71615cc, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732137383845 2024-11-20T21:16:27,260 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ab14844bcc24834ac7ae016f5b3b17e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732137383845 2024-11-20T21:16:27,261 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting e675376333e14aa49750696d0113221c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732137383904 2024-11-20T21:16:27,261 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 289b7f3a3bf14dd09b1bc5e1006a92ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732137383904 2024-11-20T21:16:27,261 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 91e70128629646be9d03e4783b38c988, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732137386081 2024-11-20T21:16:27,261 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting e61a48eadc8f4e85be992a048b132481, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732137386081 2024-11-20T21:16:27,267 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ba04bb2bd029935d159faf0939a1f26#B#compaction#539 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:27,267 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/bd3c4462c80e41f5b2b8e25218ab07ea is 50, key is test_row_0/B:col10/1732137383915/Put/seqid=0 2024-11-20T21:16:27,268 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:27,285 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120ef7471a1caf945a49e883698b5800123_2ba04bb2bd029935d159faf0939a1f26 store=[table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:27,288 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120ef7471a1caf945a49e883698b5800123_2ba04bb2bd029935d159faf0939a1f26, store=[table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:27,288 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120ef7471a1caf945a49e883698b5800123_2ba04bb2bd029935d159faf0939a1f26 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:27,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742457_1633 (size=12207) 2024-11-20T21:16:27,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742458_1634 (size=4469) 2024-11-20T21:16:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:27,552 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2ba04bb2bd029935d159faf0939a1f26 3/3 column families, dataSize=147.60 KB heapSize=420 KB 2024-11-20T21:16:27,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=A 2024-11-20T21:16:27,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T21:16:27,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=B 2024-11-20T21:16:27,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T21:16:27,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=C 2024-11-20T21:16:27,555 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-20T21:16:27,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112049ebef3120094bc7a9d9bceda8b2431d_2ba04bb2bd029935d159faf0939a1f26 is 50, key is test_row_0/A:col10/1732137386907/Put/seqid=0 2024-11-20T21:16:27,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742459_1635 (size=14794) 2024-11-20T21:16:27,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:27,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137447560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:27,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:27,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:27,566 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:27,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:27,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137447561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:27,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:27,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:27,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:27,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:27,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137447562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:27,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:27,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:27,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:27,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:27,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137447679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:27,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:27,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137447679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:27,684 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:27,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:27,685 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:27,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:27,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:27,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:27,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:27,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137447682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:27,686 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:27,686 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:27,686 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:27,706 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/bd3c4462c80e41f5b2b8e25218ab07ea as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/bd3c4462c80e41f5b2b8e25218ab07ea 2024-11-20T21:16:27,714 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2ba04bb2bd029935d159faf0939a1f26/B of 2ba04bb2bd029935d159faf0939a1f26 into bd3c4462c80e41f5b2b8e25218ab07ea(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:27,714 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:27,714 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., storeName=2ba04bb2bd029935d159faf0939a1f26/B, priority=13, startTime=1732137387253; duration=0sec 2024-11-20T21:16:27,714 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:16:27,714 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ba04bb2bd029935d159faf0939a1f26:B 2024-11-20T21:16:27,715 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:16:27,715 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:16:27,715 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 2ba04bb2bd029935d159faf0939a1f26/C is initiating minor compaction (all files) 2024-11-20T21:16:27,716 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ba04bb2bd029935d159faf0939a1f26/C in TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:27,716 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/22a9705afcb046f082e1af32447ce711, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/384107f1e76f4f6890e75031d752d6e2, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/beb960b23b4642709eec1ae58d75403a] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp, totalSize=33.0 K 2024-11-20T21:16:27,716 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 22a9705afcb046f082e1af32447ce711, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1732137383845 2024-11-20T21:16:27,717 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 384107f1e76f4f6890e75031d752d6e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732137383904 2024-11-20T21:16:27,717 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting beb960b23b4642709eec1ae58d75403a, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732137386081 2024-11-20T21:16:27,722 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ba04bb2bd029935d159faf0939a1f26#A#compaction#540 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:27,723 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/6d4ea3e16828498ba44331817bfd978b is 175, key is test_row_0/A:col10/1732137383915/Put/seqid=0 2024-11-20T21:16:27,749 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ba04bb2bd029935d159faf0939a1f26#C#compaction#542 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:27,750 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/95e2863f7ca94b1e946180e6763cffed is 50, key is test_row_0/C:col10/1732137383915/Put/seqid=0 2024-11-20T21:16:27,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742460_1636 (size=31268) 2024-11-20T21:16:27,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742461_1637 (size=12207) 2024-11-20T21:16:27,832 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/95e2863f7ca94b1e946180e6763cffed as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/95e2863f7ca94b1e946180e6763cffed 2024-11-20T21:16:27,842 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2ba04bb2bd029935d159faf0939a1f26/C of 2ba04bb2bd029935d159faf0939a1f26 into 95e2863f7ca94b1e946180e6763cffed(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:27,842 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:27,843 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., storeName=2ba04bb2bd029935d159faf0939a1f26/C, priority=13, startTime=1732137387253; duration=0sec 2024-11-20T21:16:27,843 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:27,843 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ba04bb2bd029935d159faf0939a1f26:C 2024-11-20T21:16:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-20T21:16:27,887 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:16:27,887 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:16:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-20T21:16:27,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T21:16:27,889 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:16:27,890 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:16:27,890 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:16:27,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137447887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:27,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:27,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:27,891 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:27,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:27,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137447897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:27,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:27,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:27,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:27,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:27,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137447897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:27,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:27,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:27,901 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:27,966 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:27,969 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112049ebef3120094bc7a9d9bceda8b2431d_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112049ebef3120094bc7a9d9bceda8b2431d_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:27,970 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/dc7840f137664093bf5a58bd29c8a468, store: [table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:27,971 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/dc7840f137664093bf5a58bd29c8a468 is 175, key is test_row_0/A:col10/1732137386907/Put/seqid=0 2024-11-20T21:16:27,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742462_1638 (size=39749) 2024-11-20T21:16:27,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T21:16:28,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-20T21:16:28,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:28,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:28,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:28,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:28,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:28,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:28,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:28,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44216 deadline: 1732137448086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:28,090 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 0b16ab3f13b7,36987,1732137236536 for 'test_row_0' in 2ba04bb2bd029935d159faf0939a1f26 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T21:16:28,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:28,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:28,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:28,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:28,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44310 deadline: 1732137448097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:28,100 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 0b16ab3f13b7,36987,1732137236536 for 'test_row_2' in 2ba04bb2bd029935d159faf0939a1f26 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T21:16:28,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:28,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:28,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:28,162 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/6d4ea3e16828498ba44331817bfd978b as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/6d4ea3e16828498ba44331817bfd978b 2024-11-20T21:16:28,166 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2ba04bb2bd029935d159faf0939a1f26/A of 2ba04bb2bd029935d159faf0939a1f26 into 6d4ea3e16828498ba44331817bfd978b(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:28,166 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:28,166 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., storeName=2ba04bb2bd029935d159faf0939a1f26/A, priority=13, startTime=1732137387253; duration=0sec 2024-11-20T21:16:28,166 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:28,166 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ba04bb2bd029935d159faf0939a1f26:A 2024-11-20T21:16:28,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-20T21:16:28,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:28,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:28,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:28,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:28,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:28,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:28,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:28,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137448197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:28,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:28,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T21:16:28,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:28,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137448207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:28,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:28,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:28,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:28,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137448207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:28,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:28,211 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:28,211 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:28,211 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:28,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-20T21:16:28,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:28,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:28,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:28,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:28,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:28,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:28,377 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=167, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/dc7840f137664093bf5a58bd29c8a468 2024-11-20T21:16:28,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/6d26530c4cc14bf79c7fe8554c8aa891 is 50, key is test_row_0/B:col10/1732137386907/Put/seqid=0 2024-11-20T21:16:28,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742463_1639 (size=12151) 2024-11-20T21:16:28,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-20T21:16:28,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:28,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:28,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:28,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:28,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:28,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:28,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T21:16:28,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-20T21:16:28,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:28,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:28,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:28,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:28,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:28,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137448709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:28,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:28,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:28,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:28,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:28,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137448719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:28,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:28,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:28,721 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:28,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:28,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137448721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:28,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:28,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:28,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:28,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/6d26530c4cc14bf79c7fe8554c8aa891 2024-11-20T21:16:28,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/e0094382d9784eeda9756c97eb8671ec is 50, key is test_row_0/C:col10/1732137386907/Put/seqid=0 2024-11-20T21:16:28,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-20T21:16:28,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:28,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:28,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:28,816 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:28,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:28,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:28,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742464_1640 (size=12151) 2024-11-20T21:16:28,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/e0094382d9784eeda9756c97eb8671ec 2024-11-20T21:16:28,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/dc7840f137664093bf5a58bd29c8a468 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/dc7840f137664093bf5a58bd29c8a468 2024-11-20T21:16:28,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/dc7840f137664093bf5a58bd29c8a468, entries=200, sequenceid=167, filesize=38.8 K 2024-11-20T21:16:28,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/6d26530c4cc14bf79c7fe8554c8aa891 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/6d26530c4cc14bf79c7fe8554c8aa891 2024-11-20T21:16:28,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/6d26530c4cc14bf79c7fe8554c8aa891, entries=150, sequenceid=167, filesize=11.9 K 2024-11-20T21:16:28,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/e0094382d9784eeda9756c97eb8671ec as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/e0094382d9784eeda9756c97eb8671ec 2024-11-20T21:16:28,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/e0094382d9784eeda9756c97eb8671ec, entries=150, sequenceid=167, filesize=11.9 K 2024-11-20T21:16:28,908 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~459.80 KB/470832, currentSize=20.13 KB/20610 for 2ba04bb2bd029935d159faf0939a1f26 in 1357ms, sequenceid=167, compaction requested=false 2024-11-20T21:16:28,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:28,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-20T21:16:28,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:28,991 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 2ba04bb2bd029935d159faf0939a1f26 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-20T21:16:28,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=A 2024-11-20T21:16:28,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:28,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=B 2024-11-20T21:16:28,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:28,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=C 2024-11-20T21:16:28,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:28,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:28,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120eb716e36a19a4dad9be51826c69412b9_2ba04bb2bd029935d159faf0939a1f26 is 50, key is test_row_1/A:col10/1732137387559/Put/seqid=0 2024-11-20T21:16:29,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T21:16:29,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742465_1641 (size=9814) 2024-11-20T21:16:29,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,055 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120eb716e36a19a4dad9be51826c69412b9_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120eb716e36a19a4dad9be51826c69412b9_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:29,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/b937926ece0f489a9f054e9729f2662a, store: [table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:29,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/b937926ece0f489a9f054e9729f2662a is 175, key is test_row_1/A:col10/1732137387559/Put/seqid=0 2024-11-20T21:16:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742466_1642 (size=22461) 2024-11-20T21:16:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,509 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=176, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/b937926ece0f489a9f054e9729f2662a 2024-11-20T21:16:29,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/cdd671a7b0c646efac8ab8c01f67e10b is 50, key is test_row_1/B:col10/1732137387559/Put/seqid=0 2024-11-20T21:16:29,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742467_1643 (size=9757) 2024-11-20T21:16:29,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:29,743 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:29,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:29,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732137389765, max timestamp=1732137389778Num uniques 859; 2024-11-20T21:16:29,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T21:16:29,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732137389765, max timestamp=1732137389778Num uniques 859; 2024-11-20T21:16:29,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T21:16:29,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1732137389765, max timestamp=1732137389778Num uniques 859; 2024-11-20T21:16:29,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-20T21:16:29,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7686, totalHeapSize=17712, min timestamp=1732137389815, max timestamp=1732137389843Num uniques 1118; 2024-11-20T21:16:29,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51196, new segment data size=7686, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-20T21:16:29,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1732137389815, max timestamp=1732137389843Num uniques 1118; 2024-11-20T21:16:29,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-11-20T21:16:29,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732137389815, max timestamp=1732137389843Num uniques 1118; 2024-11-20T21:16:29,850 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=6870, suffix heap size=144240,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=150 2024-11-20T21:16:29,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1732137389888, max timestamp=1732137389899Num uniques 1027; 2024-11-20T21:16:29,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-20T21:16:29,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732137389888, max timestamp=1732137389899Num uniques 1009; 2024-11-20T21:16:29,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-11-20T21:16:29,910 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=174, cellSize=7964, totalHeapSize=18336, min timestamp=1732137389888, max timestamp=1732137389900Num uniques 1027; 2024-11-20T21:16:29,910 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47029, new segment data size=7964, suffix heap size=134776,new segment heap size=18336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=174 2024-11-20T21:16:29,961 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/cdd671a7b0c646efac8ab8c01f67e10b 2024-11-20T21:16:29,963 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732137389943, max timestamp=1732137389958Num uniques 1033; 2024-11-20T21:16:29,963 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47311, new segment data size=10806, suffix heap size=135400,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1033, new segment cells count=236 2024-11-20T21:16:29,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732137389943, max timestamp=1732137389958Num uniques 1036; 2024-11-20T21:16:29,964 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-20T21:16:29,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1732137389943, max timestamp=1732137389958Num uniques 1009; 2024-11-20T21:16:29,967 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10806, suffix heap size=132904,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=236 2024-11-20T21:16:29,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/0ce2be6a5d354e10a5f57babc99af44a is 50, key is test_row_1/C:col10/1732137387559/Put/seqid=0 2024-11-20T21:16:30,022 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732137390006, max timestamp=1732137390017Num uniques 1095; 2024-11-20T21:16:30,022 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-20T21:16:30,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1732137390006, max timestamp=1732137390020Num uniques 1095; 2024-11-20T21:16:30,025 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-20T21:16:30,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1732137390006, max timestamp=1732137390017Num uniques 1095; 2024-11-20T21:16:30,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-20T21:16:30,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T21:16:30,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742468_1644 (size=9757) 2024-11-20T21:16:30,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732137390045, max timestamp=1732137390071Num uniques 1104; 2024-11-20T21:16:30,076 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-20T21:16:30,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732137390045, max timestamp=1732137390071Num uniques 1054; 2024-11-20T21:16:30,080 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=6870, suffix heap size=137584,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=150 2024-11-20T21:16:30,077 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1732137390045, max timestamp=1732137390071Num uniques 1104; 2024-11-20T21:16:30,087 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-20T21:16:30,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732137390124, max timestamp=1732137390133Num uniques 1063; 2024-11-20T21:16:30,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-20T21:16:30,144 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1732137390124, max timestamp=1732137390133Num uniques 1063; 2024-11-20T21:16:30,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=7458, suffix heap size=138520,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=163 2024-11-20T21:16:30,146 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732137390124, max timestamp=1732137390133Num uniques 1009; 2024-11-20T21:16:30,146 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46208, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-11-20T21:16:30,196 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732137390171, max timestamp=1732137390192Num uniques 1022; 2024-11-20T21:16:30,197 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-20T21:16:30,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732137390171, max timestamp=1732137390192Num uniques 1009; 2024-11-20T21:16:30,202 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=6870, suffix heap size=132904,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=150 2024-11-20T21:16:30,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1732137390171, max timestamp=1732137390192Num uniques 1022; 2024-11-20T21:16:30,206 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=7872, suffix heap size=134256,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=172 2024-11-20T21:16:30,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1732137390232, max timestamp=1732137390239Num uniques 1031; 2024-11-20T21:16:30,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-20T21:16:30,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1732137390232, max timestamp=1732137390239Num uniques 1031; 2024-11-20T21:16:30,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-20T21:16:30,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1732137390232, max timestamp=1732137390239Num uniques 1009; 2024-11-20T21:16:30,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8286, suffix heap size=132904,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=181 2024-11-20T21:16:30,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1732137390269, max timestamp=1732137390281Num uniques 1040; 2024-11-20T21:16:30,286 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-20T21:16:30,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732137390269, max timestamp=1732137390281Num uniques 1040; 2024-11-20T21:16:30,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=6870, suffix heap size=136128,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=150 2024-11-20T21:16:30,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1732137390269, max timestamp=1732137390281Num uniques 1040; 2024-11-20T21:16:30,292 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=6870, suffix heap size=136128,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=150 2024-11-20T21:16:30,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1732137390311, max timestamp=1732137390323Num uniques 1009; 2024-11-20T21:16:30,328 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=11404, suffix heap size=132904,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=249 2024-11-20T21:16:30,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1732137390311, max timestamp=1732137390323Num uniques 1009; 2024-11-20T21:16:30,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=11404, suffix heap size=132904,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=249 2024-11-20T21:16:30,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1732137390311, max timestamp=1732137390323Num uniques 1049; 2024-11-20T21:16:30,333 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=11404, suffix heap size=137064,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=249 2024-11-20T21:16:30,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1732137390344, max timestamp=1732137390375Num uniques 1108; 2024-11-20T21:16:30,382 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=7230, suffix heap size=143200,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=158 2024-11-20T21:16:30,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1732137390344, max timestamp=1732137390375Num uniques 1108; 2024-11-20T21:16:30,384 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=7230, suffix heap size=143200,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=158 2024-11-20T21:16:30,385 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1732137390344, max timestamp=1732137390375Num uniques 1108; 2024-11-20T21:16:30,386 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50740, new segment data size=7230, suffix heap size=143200,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1108, new segment cells count=158 2024-11-20T21:16:30,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=267, cellSize=12222, totalHeapSize=28008, min timestamp=1732137390382, max timestamp=1732137390427Num uniques 1017; 2024-11-20T21:16:30,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=12222, suffix heap size=133736,new segment heap size=28008 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=267 2024-11-20T21:16:30,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=267, cellSize=12222, totalHeapSize=28008, min timestamp=1732137390382, max timestamp=1732137390427Num uniques 1017; 2024-11-20T21:16:30,431 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=12222, suffix heap size=133736,new segment heap size=28008 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=267 2024-11-20T21:16:30,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=267, cellSize=12222, totalHeapSize=28008, min timestamp=1732137390382, max timestamp=1732137390427Num uniques 1017; 2024-11-20T21:16:30,432 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/0b16ab3f13b7:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=12222, suffix heap size=133736,new segment heap size=28008 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=267 2024-11-20T21:16:30,437 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/0ce2be6a5d354e10a5f57babc99af44a 2024-11-20T21:16:30,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/b937926ece0f489a9f054e9729f2662a as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/b937926ece0f489a9f054e9729f2662a 2024-11-20T21:16:30,446 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/b937926ece0f489a9f054e9729f2662a, entries=100, sequenceid=176, filesize=21.9 K 2024-11-20T21:16:30,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/cdd671a7b0c646efac8ab8c01f67e10b as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/cdd671a7b0c646efac8ab8c01f67e10b 2024-11-20T21:16:30,450 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/cdd671a7b0c646efac8ab8c01f67e10b, entries=100, sequenceid=176, filesize=9.5 K 2024-11-20T21:16:30,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/0ce2be6a5d354e10a5f57babc99af44a as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/0ce2be6a5d354e10a5f57babc99af44a 2024-11-20T21:16:30,455 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/0ce2be6a5d354e10a5f57babc99af44a, entries=100, sequenceid=176, filesize=9.5 K 2024-11-20T21:16:30,456 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=107.34 KB/109920 for 2ba04bb2bd029935d159faf0939a1f26 in 1465ms, sequenceid=176, compaction requested=true 2024-11-20T21:16:30,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:30,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:30,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-20T21:16:30,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-20T21:16:30,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:30,459 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-20T21:16:30,459 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5670 sec 2024-11-20T21:16:30,460 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2ba04bb2bd029935d159faf0939a1f26 3/3 column families, dataSize=114.05 KB heapSize=315.09 KB 2024-11-20T21:16:30,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=A 2024-11-20T21:16:30,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:16:30,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=B 2024-11-20T21:16:30,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:16:30,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=C 2024-11-20T21:16:30,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-20T21:16:30,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.5730 sec 2024-11-20T21:16:30,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120fe4c42c0cfbe4faa8140ca59493dda36_2ba04bb2bd029935d159faf0939a1f26 is 50, key is test_row_0/A:col10/1732137390453/Put/seqid=0 2024-11-20T21:16:30,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742469_1645 (size=14994) 2024-11-20T21:16:30,475 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:30,481 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120fe4c42c0cfbe4faa8140ca59493dda36_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120fe4c42c0cfbe4faa8140ca59493dda36_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:30,484 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/4150d4c5cafd4bd197f5a311bec8bb5d, store: [table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:30,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/4150d4c5cafd4bd197f5a311bec8bb5d is 175, key is test_row_0/A:col10/1732137390453/Put/seqid=0 2024-11-20T21:16:30,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742470_1646 (size=39949) 2024-11-20T21:16:30,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:30,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137450486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:30,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:30,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:30,515 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:30,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:30,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137450517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:30,524 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:30,524 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:30,525 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:30,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:30,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137450524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:30,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:30,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:30,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:30,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:30,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137450627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:30,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:30,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:30,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:30,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137450627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:30,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:30,634 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:30,634 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:30,634 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:30,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:30,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137450637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:30,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:30,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:30,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:30,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:30,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:30,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137450837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:30,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137450836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:30,843 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:30,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:30,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137450847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:30,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:30,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:30,852 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:30,904 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=414, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/4150d4c5cafd4bd197f5a311bec8bb5d 2024-11-20T21:16:30,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/c7d4b587c48a4f9fac3cc2c6e34a9e4d is 50, key is test_row_0/B:col10/1732137390453/Put/seqid=0 2024-11-20T21:16:30,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742471_1647 (size=12301) 2024-11-20T21:16:30,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/c7d4b587c48a4f9fac3cc2c6e34a9e4d 2024-11-20T21:16:30,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/5e9de55bc50746da9b83c828a95d988b is 50, key is test_row_0/C:col10/1732137390453/Put/seqid=0 2024-11-20T21:16:30,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742472_1648 (size=12301) 2024-11-20T21:16:31,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137451147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:31,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:31,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:31,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:31,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137451147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:31,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:31,152 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:31,152 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:31,152 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:31,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:31,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137451157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:31,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:31,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:31,161 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:31,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/5e9de55bc50746da9b83c828a95d988b 2024-11-20T21:16:31,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/4150d4c5cafd4bd197f5a311bec8bb5d as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/4150d4c5cafd4bd197f5a311bec8bb5d 2024-11-20T21:16:31,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/4150d4c5cafd4bd197f5a311bec8bb5d, entries=200, sequenceid=414, filesize=39.0 K 2024-11-20T21:16:31,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/c7d4b587c48a4f9fac3cc2c6e34a9e4d as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/c7d4b587c48a4f9fac3cc2c6e34a9e4d 2024-11-20T21:16:31,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/c7d4b587c48a4f9fac3cc2c6e34a9e4d, entries=150, sequenceid=414, filesize=12.0 K 2024-11-20T21:16:31,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/5e9de55bc50746da9b83c828a95d988b as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/5e9de55bc50746da9b83c828a95d988b 2024-11-20T21:16:31,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/5e9de55bc50746da9b83c828a95d988b, entries=150, sequenceid=414, filesize=12.0 K 2024-11-20T21:16:31,350 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~315.05 KB/322608, currentSize=73.80 KB/75570 for 2ba04bb2bd029935d159faf0939a1f26 in 891ms, sequenceid=414, compaction requested=true 2024-11-20T21:16:31,350 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:31,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ba04bb2bd029935d159faf0939a1f26:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:16:31,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:31,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ba04bb2bd029935d159faf0939a1f26:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:16:31,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:31,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ba04bb2bd029935d159faf0939a1f26:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:16:31,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:16:31,350 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:16:31,350 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:16:31,351 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133427 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:16:31,351 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 2ba04bb2bd029935d159faf0939a1f26/A is initiating minor compaction (all files) 2024-11-20T21:16:31,351 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ba04bb2bd029935d159faf0939a1f26/A in TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:31,352 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/6d4ea3e16828498ba44331817bfd978b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/dc7840f137664093bf5a58bd29c8a468, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/b937926ece0f489a9f054e9729f2662a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/4150d4c5cafd4bd197f5a311bec8bb5d] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp, totalSize=130.3 K 2024-11-20T21:16:31,352 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:31,352 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. files: [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/6d4ea3e16828498ba44331817bfd978b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/dc7840f137664093bf5a58bd29c8a468, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/b937926ece0f489a9f054e9729f2662a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/4150d4c5cafd4bd197f5a311bec8bb5d] 2024-11-20T21:16:31,354 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d4ea3e16828498ba44331817bfd978b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732137383915 2024-11-20T21:16:31,354 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46416 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:16:31,354 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 2ba04bb2bd029935d159faf0939a1f26/B is initiating minor compaction (all files) 2024-11-20T21:16:31,354 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ba04bb2bd029935d159faf0939a1f26/B in TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:31,354 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/bd3c4462c80e41f5b2b8e25218ab07ea, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/6d26530c4cc14bf79c7fe8554c8aa891, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/cdd671a7b0c646efac8ab8c01f67e10b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/c7d4b587c48a4f9fac3cc2c6e34a9e4d] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp, totalSize=45.3 K 2024-11-20T21:16:31,355 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc7840f137664093bf5a58bd29c8a468, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732137386907 2024-11-20T21:16:31,355 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting bd3c4462c80e41f5b2b8e25218ab07ea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732137383915 2024-11-20T21:16:31,355 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d26530c4cc14bf79c7fe8554c8aa891, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732137386907 2024-11-20T21:16:31,355 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting b937926ece0f489a9f054e9729f2662a, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732137387559 2024-11-20T21:16:31,356 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4150d4c5cafd4bd197f5a311bec8bb5d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732137390438 2024-11-20T21:16:31,356 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting cdd671a7b0c646efac8ab8c01f67e10b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732137387559 2024-11-20T21:16:31,356 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting c7d4b587c48a4f9fac3cc2c6e34a9e4d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732137390438 2024-11-20T21:16:31,366 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:31,369 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411207dfb9c82d9be4eb79cc6e1aadcf3188d_2ba04bb2bd029935d159faf0939a1f26 store=[table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:31,370 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ba04bb2bd029935d159faf0939a1f26#B#compaction#552 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:31,371 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/22d9213a531448998c489da1f81abf58 is 50, key is test_row_0/B:col10/1732137390453/Put/seqid=0 2024-11-20T21:16:31,372 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411207dfb9c82d9be4eb79cc6e1aadcf3188d_2ba04bb2bd029935d159faf0939a1f26, store=[table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:31,372 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207dfb9c82d9be4eb79cc6e1aadcf3188d_2ba04bb2bd029935d159faf0939a1f26 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:31,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742473_1649 (size=12643) 2024-11-20T21:16:31,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742474_1650 (size=4469) 2024-11-20T21:16:31,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:31,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2ba04bb2bd029935d159faf0939a1f26 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-20T21:16:31,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=A 2024-11-20T21:16:31,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:31,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=B 2024-11-20T21:16:31,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:31,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=C 2024-11-20T21:16:31,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:31,667 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a9a3af9efeba4b949ad77e48bcbd35a3_2ba04bb2bd029935d159faf0939a1f26 is 50, key is test_row_0/A:col10/1732137390485/Put/seqid=0 2024-11-20T21:16:31,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742475_1651 (size=14994) 2024-11-20T21:16:31,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:31,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137451693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:31,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:31,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:31,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:31,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:31,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137451694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:31,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:31,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:31,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:31,701 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:31,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137451694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:31,701 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:31,701 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:31,701 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:31,778 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ba04bb2bd029935d159faf0939a1f26#A#compaction#551 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:31,779 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/40c0355cfba146dd83a6b02e4543fd0d is 175, key is test_row_0/A:col10/1732137390453/Put/seqid=0 2024-11-20T21:16:31,781 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/22d9213a531448998c489da1f81abf58 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/22d9213a531448998c489da1f81abf58 2024-11-20T21:16:31,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742476_1652 (size=31597) 2024-11-20T21:16:31,785 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2ba04bb2bd029935d159faf0939a1f26/B of 2ba04bb2bd029935d159faf0939a1f26 into 22d9213a531448998c489da1f81abf58(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:31,785 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:31,785 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., storeName=2ba04bb2bd029935d159faf0939a1f26/B, priority=12, startTime=1732137391350; duration=0sec 2024-11-20T21:16:31,785 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:16:31,785 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ba04bb2bd029935d159faf0939a1f26:B 2024-11-20T21:16:31,785 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:16:31,786 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46416 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:16:31,786 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 2ba04bb2bd029935d159faf0939a1f26/C is initiating minor compaction (all files) 2024-11-20T21:16:31,786 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ba04bb2bd029935d159faf0939a1f26/C in TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:31,786 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/95e2863f7ca94b1e946180e6763cffed, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/e0094382d9784eeda9756c97eb8671ec, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/0ce2be6a5d354e10a5f57babc99af44a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/5e9de55bc50746da9b83c828a95d988b] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp, totalSize=45.3 K 2024-11-20T21:16:31,786 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 95e2863f7ca94b1e946180e6763cffed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1732137383915 2024-11-20T21:16:31,787 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting e0094382d9784eeda9756c97eb8671ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732137386907 2024-11-20T21:16:31,787 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ce2be6a5d354e10a5f57babc99af44a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1732137387559 2024-11-20T21:16:31,787 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e9de55bc50746da9b83c828a95d988b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732137390438 2024-11-20T21:16:31,795 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ba04bb2bd029935d159faf0939a1f26#C#compaction#554 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:31,796 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/d49ef2c917ab4dc4b15d59480e47126c is 50, key is test_row_0/C:col10/1732137390453/Put/seqid=0 2024-11-20T21:16:31,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742477_1653 (size=12643) 2024-11-20T21:16:31,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:31,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137451807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:31,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:31,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137451807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:31,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:31,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:31,811 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:31,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:31,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137451807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:31,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:31,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:31,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:31,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:31,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:31,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:32,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:32,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137452016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:32,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:32,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:32,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137452017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:32,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:32,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137452017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:32,020 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:32,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-20T21:16:32,057 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:16:32,058 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:16:32,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-20T21:16:32,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-20T21:16:32,059 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:16:32,060 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:16:32,060 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:16:32,072 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:32,075 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120a9a3af9efeba4b949ad77e48bcbd35a3_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a9a3af9efeba4b949ad77e48bcbd35a3_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:32,075 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/f24bb7ced89a4d1092599fe63ca1dcbd, store: [table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:32,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/f24bb7ced89a4d1092599fe63ca1dcbd is 175, key is test_row_0/A:col10/1732137390485/Put/seqid=0 2024-11-20T21:16:32,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742478_1654 (size=39949) 2024-11-20T21:16:32,079 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=429, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/f24bb7ced89a4d1092599fe63ca1dcbd 2024-11-20T21:16:32,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/3488f32b3a8e48ab9fe0a5573130c98c is 50, key is test_row_0/B:col10/1732137390485/Put/seqid=0 2024-11-20T21:16:32,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742479_1655 (size=12301) 2024-11-20T21:16:32,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:32,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44216 deadline: 1732137452127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:32,130 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 0b16ab3f13b7,36987,1732137236536 for 'test_row_0' in 2ba04bb2bd029935d159faf0939a1f26 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T21:16:32,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:32,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:32,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:32,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:32,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44310 deadline: 1732137452147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:32,150 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to 0b16ab3f13b7,36987,1732137236536 for 'test_row_2' in 2ba04bb2bd029935d159faf0939a1f26 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-20T21:16:32,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:32,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:32,151 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:32,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-20T21:16:32,186 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/40c0355cfba146dd83a6b02e4543fd0d as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/40c0355cfba146dd83a6b02e4543fd0d 2024-11-20T21:16:32,190 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2ba04bb2bd029935d159faf0939a1f26/A of 2ba04bb2bd029935d159faf0939a1f26 into 40c0355cfba146dd83a6b02e4543fd0d(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:32,190 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:32,190 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., storeName=2ba04bb2bd029935d159faf0939a1f26/A, priority=12, startTime=1732137391350; duration=0sec 2024-11-20T21:16:32,190 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:32,190 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ba04bb2bd029935d159faf0939a1f26:A 2024-11-20T21:16:32,207 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/d49ef2c917ab4dc4b15d59480e47126c as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/d49ef2c917ab4dc4b15d59480e47126c 2024-11-20T21:16:32,211 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2ba04bb2bd029935d159faf0939a1f26/C of 2ba04bb2bd029935d159faf0939a1f26 into d49ef2c917ab4dc4b15d59480e47126c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:32,211 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:32,211 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., storeName=2ba04bb2bd029935d159faf0939a1f26/C, priority=12, startTime=1732137391350; duration=0sec 2024-11-20T21:16:32,211 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:32,211 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ba04bb2bd029935d159faf0939a1f26:C 2024-11-20T21:16:32,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-20T21:16:32,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:32,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:32,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:32,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:32,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:32,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:32,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:32,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137452327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:32,330 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:32,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:32,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:32,331 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:32,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137452327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:32,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:32,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137452327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:32,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:32,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:32,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:32,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:32,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:32,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:32,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-20T21:16:32,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:32,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:32,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:32,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:32,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:32,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:32,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-20T21:16:32,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/3488f32b3a8e48ab9fe0a5573130c98c 2024-11-20T21:16:32,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/8adee7b6c70a40c68b435f0c9b339e71 is 50, key is test_row_0/C:col10/1732137390485/Put/seqid=0 2024-11-20T21:16:32,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742480_1656 (size=12301) 2024-11-20T21:16:32,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-20T21:16:32,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:32,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:32,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:32,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:32,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:32,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:32,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-20T21:16:32,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:32,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:32,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:32,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:32,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:32,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:32,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-20T21:16:32,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-20T21:16:32,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:32,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:32,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:32,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:32,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:32,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:32,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:32,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137452845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:32,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:32,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:32,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:32,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:32,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137452845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:32,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:32,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137452845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:32,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:32,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:32,849 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:32,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:32,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:32,849 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:32,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/8adee7b6c70a40c68b435f0c9b339e71 2024-11-20T21:16:32,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/f24bb7ced89a4d1092599fe63ca1dcbd as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/f24bb7ced89a4d1092599fe63ca1dcbd 2024-11-20T21:16:32,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/f24bb7ced89a4d1092599fe63ca1dcbd, entries=200, sequenceid=429, filesize=39.0 K 2024-11-20T21:16:32,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/3488f32b3a8e48ab9fe0a5573130c98c as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/3488f32b3a8e48ab9fe0a5573130c98c 2024-11-20T21:16:32,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/3488f32b3a8e48ab9fe0a5573130c98c, entries=150, sequenceid=429, filesize=12.0 K 2024-11-20T21:16:32,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/8adee7b6c70a40c68b435f0c9b339e71 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/8adee7b6c70a40c68b435f0c9b339e71 2024-11-20T21:16:32,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/8adee7b6c70a40c68b435f0c9b339e71, entries=150, sequenceid=429, filesize=12.0 K 2024-11-20T21:16:32,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 2ba04bb2bd029935d159faf0939a1f26 in 1270ms, sequenceid=429, compaction requested=false 2024-11-20T21:16:32,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:32,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-20T21:16:32,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:32,976 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 2ba04bb2bd029935d159faf0939a1f26 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T21:16:32,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=A 2024-11-20T21:16:32,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:32,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=B 2024-11-20T21:16:32,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:32,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=C 2024-11-20T21:16:32,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:32,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112036a20ad83fc74b2d9c8ec3eb072149a6_2ba04bb2bd029935d159faf0939a1f26 is 50, key is test_row_0/A:col10/1732137391693/Put/seqid=0 2024-11-20T21:16:32,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742481_1657 (size=12454) 2024-11-20T21:16:33,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-20T21:16:33,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:33,399 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112036a20ad83fc74b2d9c8ec3eb072149a6_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112036a20ad83fc74b2d9c8ec3eb072149a6_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:33,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/4e79ec6cee8e474cae62887eae16aa48, store: [table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:33,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/4e79ec6cee8e474cae62887eae16aa48 is 175, key is test_row_0/A:col10/1732137391693/Put/seqid=0 2024-11-20T21:16:33,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742482_1658 (size=31255) 2024-11-20T21:16:33,806 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=449, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/4e79ec6cee8e474cae62887eae16aa48 2024-11-20T21:16:33,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/8c7d879a1567425b897b7ca90a73aadc is 50, key is test_row_0/B:col10/1732137391693/Put/seqid=0 2024-11-20T21:16:33,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742483_1659 (size=12301) 2024-11-20T21:16:33,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:33,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:33,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:33,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137453883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:33,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:33,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137453884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:33,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:33,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137453885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:33,890 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:34,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:34,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137453997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:34,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:34,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137453997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:34,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:34,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137453997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:34,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:34,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:34,001 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:34,002 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:34,002 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:34,002 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:34,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:34,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:34,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:34,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-20T21:16:34,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:34,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137454208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:34,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:34,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:34,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:34,210 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:34,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137454208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:34,210 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:34,210 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:34,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:34,210 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:34,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137454208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:34,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:34,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:34,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:34,218 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/8c7d879a1567425b897b7ca90a73aadc 2024-11-20T21:16:34,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/e06bbfa0b7d542fb8262eeb97e2aa4ca is 50, key is test_row_0/C:col10/1732137391693/Put/seqid=0 2024-11-20T21:16:34,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742484_1660 (size=12301) 2024-11-20T21:16:34,256 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=449 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/e06bbfa0b7d542fb8262eeb97e2aa4ca 2024-11-20T21:16:34,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/4e79ec6cee8e474cae62887eae16aa48 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/4e79ec6cee8e474cae62887eae16aa48 2024-11-20T21:16:34,268 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/4e79ec6cee8e474cae62887eae16aa48, entries=150, sequenceid=449, filesize=30.5 K 2024-11-20T21:16:34,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/8c7d879a1567425b897b7ca90a73aadc as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/8c7d879a1567425b897b7ca90a73aadc 2024-11-20T21:16:34,273 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/8c7d879a1567425b897b7ca90a73aadc, entries=150, sequenceid=449, filesize=12.0 K 2024-11-20T21:16:34,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/e06bbfa0b7d542fb8262eeb97e2aa4ca as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/e06bbfa0b7d542fb8262eeb97e2aa4ca 2024-11-20T21:16:34,277 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/e06bbfa0b7d542fb8262eeb97e2aa4ca, entries=150, sequenceid=449, filesize=12.0 K 2024-11-20T21:16:34,279 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 2ba04bb2bd029935d159faf0939a1f26 in 1302ms, sequenceid=449, compaction requested=true 2024-11-20T21:16:34,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:34,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:34,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-20T21:16:34,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-20T21:16:34,283 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-20T21:16:34,283 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2210 sec 2024-11-20T21:16:34,286 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.2270 sec 2024-11-20T21:16:34,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:34,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2ba04bb2bd029935d159faf0939a1f26 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T21:16:34,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=A 2024-11-20T21:16:34,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:34,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=B 2024-11-20T21:16:34,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:34,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=C 2024-11-20T21:16:34,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:34,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112029219ade792444dcb5825ad0c1f2c807_2ba04bb2bd029935d159faf0939a1f26 is 50, key is test_row_0/A:col10/1732137393883/Put/seqid=0 2024-11-20T21:16:34,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742485_1661 (size=12454) 2024-11-20T21:16:34,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:34,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137454539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:34,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:34,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:34,545 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:34,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:34,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137454542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:34,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:34,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:34,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:34,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:34,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137454545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:34,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:34,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:34,550 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:34,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:34,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137454657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:34,662 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:34,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:34,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:34,662 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:34,662 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:34,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137454657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:34,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137454657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:34,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:34,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:34,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:34,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:34,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:34,662 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:34,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:34,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137454868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:34,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:34,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:34,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137454867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:34,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:34,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:34,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:34,871 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:34,871 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:34,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137454868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:34,871 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:34,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:34,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:34,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:34,933 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:34,941 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112029219ade792444dcb5825ad0c1f2c807_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112029219ade792444dcb5825ad0c1f2c807_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:34,942 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/2670f1dcd243477aa8fb124b75157d40, store: [table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:34,942 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/2670f1dcd243477aa8fb124b75157d40 is 175, key is test_row_0/A:col10/1732137393883/Put/seqid=0 2024-11-20T21:16:34,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742486_1662 (size=31255) 2024-11-20T21:16:34,957 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=465, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/2670f1dcd243477aa8fb124b75157d40 2024-11-20T21:16:34,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/7065ddc035c04b88ad0f46a43d0f7271 is 50, key is test_row_0/B:col10/1732137393883/Put/seqid=0 2024-11-20T21:16:34,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742487_1663 (size=12301) 2024-11-20T21:16:35,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:35,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137455177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:35,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:35,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137455177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:35,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:35,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137455177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:35,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:35,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:35,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:35,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:35,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:35,183 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:35,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/7065ddc035c04b88ad0f46a43d0f7271 2024-11-20T21:16:35,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/2bf71d53c909416c8e6626b0d0c176f9 is 50, key is test_row_0/C:col10/1732137393883/Put/seqid=0 2024-11-20T21:16:35,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742488_1664 (size=12301) 2024-11-20T21:16:35,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:35,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137455687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:35,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:35,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:35,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:35,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137455697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:35,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:35,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:35,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:35,701 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:35,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137455697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:35,702 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:35,702 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:35,702 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:35,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/2bf71d53c909416c8e6626b0d0c176f9 2024-11-20T21:16:35,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/2670f1dcd243477aa8fb124b75157d40 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/2670f1dcd243477aa8fb124b75157d40 2024-11-20T21:16:35,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/2670f1dcd243477aa8fb124b75157d40, entries=150, sequenceid=465, filesize=30.5 K 2024-11-20T21:16:35,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/7065ddc035c04b88ad0f46a43d0f7271 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/7065ddc035c04b88ad0f46a43d0f7271 2024-11-20T21:16:35,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/7065ddc035c04b88ad0f46a43d0f7271, entries=150, sequenceid=465, filesize=12.0 K 2024-11-20T21:16:35,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/2bf71d53c909416c8e6626b0d0c176f9 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/2bf71d53c909416c8e6626b0d0c176f9 2024-11-20T21:16:35,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/2bf71d53c909416c8e6626b0d0c176f9, entries=150, sequenceid=465, filesize=12.0 K 2024-11-20T21:16:35,841 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 2ba04bb2bd029935d159faf0939a1f26 in 1321ms, sequenceid=465, compaction requested=true 2024-11-20T21:16:35,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:35,842 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:16:35,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ba04bb2bd029935d159faf0939a1f26:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:16:35,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:35,842 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:16:35,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ba04bb2bd029935d159faf0939a1f26:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:16:35,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:35,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ba04bb2bd029935d159faf0939a1f26:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:16:35,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:16:35,843 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:16:35,843 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 2ba04bb2bd029935d159faf0939a1f26/A is initiating minor compaction (all files) 2024-11-20T21:16:35,843 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ba04bb2bd029935d159faf0939a1f26/A in TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:35,843 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/40c0355cfba146dd83a6b02e4543fd0d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/f24bb7ced89a4d1092599fe63ca1dcbd, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/4e79ec6cee8e474cae62887eae16aa48, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/2670f1dcd243477aa8fb124b75157d40] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp, totalSize=130.9 K 2024-11-20T21:16:35,843 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:35,843 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. files: [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/40c0355cfba146dd83a6b02e4543fd0d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/f24bb7ced89a4d1092599fe63ca1dcbd, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/4e79ec6cee8e474cae62887eae16aa48, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/2670f1dcd243477aa8fb124b75157d40] 2024-11-20T21:16:35,844 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40c0355cfba146dd83a6b02e4543fd0d, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732137390438 2024-11-20T21:16:35,844 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f24bb7ced89a4d1092599fe63ca1dcbd, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732137390473 2024-11-20T21:16:35,844 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e79ec6cee8e474cae62887eae16aa48, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1732137391684 2024-11-20T21:16:35,845 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2670f1dcd243477aa8fb124b75157d40, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732137393881 2024-11-20T21:16:35,853 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49546 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:16:35,853 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 2ba04bb2bd029935d159faf0939a1f26/B is initiating minor compaction (all files) 2024-11-20T21:16:35,853 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ba04bb2bd029935d159faf0939a1f26/B in TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:35,853 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/22d9213a531448998c489da1f81abf58, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/3488f32b3a8e48ab9fe0a5573130c98c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/8c7d879a1567425b897b7ca90a73aadc, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/7065ddc035c04b88ad0f46a43d0f7271] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp, totalSize=48.4 K 2024-11-20T21:16:35,856 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 22d9213a531448998c489da1f81abf58, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732137390438 2024-11-20T21:16:35,857 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 3488f32b3a8e48ab9fe0a5573130c98c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732137390473 2024-11-20T21:16:35,857 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c7d879a1567425b897b7ca90a73aadc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1732137391684 2024-11-20T21:16:35,861 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 7065ddc035c04b88ad0f46a43d0f7271, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732137393881 2024-11-20T21:16:35,872 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:35,885 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411207299ff535d214aeba71a0106d6aebff0_2ba04bb2bd029935d159faf0939a1f26 store=[table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:35,888 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411207299ff535d214aeba71a0106d6aebff0_2ba04bb2bd029935d159faf0939a1f26, store=[table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:35,888 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411207299ff535d214aeba71a0106d6aebff0_2ba04bb2bd029935d159faf0939a1f26 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:35,894 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ba04bb2bd029935d159faf0939a1f26#B#compaction#564 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:35,894 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/0ef4640f4d6b4bdabccf244a638d251b is 50, key is test_row_0/B:col10/1732137393883/Put/seqid=0 2024-11-20T21:16:35,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742489_1665 (size=4469) 2024-11-20T21:16:35,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742490_1666 (size=12779) 2024-11-20T21:16:35,940 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/0ef4640f4d6b4bdabccf244a638d251b as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/0ef4640f4d6b4bdabccf244a638d251b 2024-11-20T21:16:35,946 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2ba04bb2bd029935d159faf0939a1f26/B of 2ba04bb2bd029935d159faf0939a1f26 into 0ef4640f4d6b4bdabccf244a638d251b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:35,946 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:35,947 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., storeName=2ba04bb2bd029935d159faf0939a1f26/B, priority=12, startTime=1732137395842; duration=0sec 2024-11-20T21:16:35,947 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:16:35,947 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ba04bb2bd029935d159faf0939a1f26:B 2024-11-20T21:16:35,947 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:16:35,948 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49546 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:16:35,948 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 2ba04bb2bd029935d159faf0939a1f26/C is initiating minor compaction (all files) 2024-11-20T21:16:35,948 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ba04bb2bd029935d159faf0939a1f26/C in TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:35,948 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/d49ef2c917ab4dc4b15d59480e47126c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/8adee7b6c70a40c68b435f0c9b339e71, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/e06bbfa0b7d542fb8262eeb97e2aa4ca, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/2bf71d53c909416c8e6626b0d0c176f9] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp, totalSize=48.4 K 2024-11-20T21:16:35,949 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting d49ef2c917ab4dc4b15d59480e47126c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1732137390438 2024-11-20T21:16:35,949 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 8adee7b6c70a40c68b435f0c9b339e71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1732137390473 2024-11-20T21:16:35,949 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting e06bbfa0b7d542fb8262eeb97e2aa4ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=449, earliestPutTs=1732137391684 2024-11-20T21:16:35,949 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bf71d53c909416c8e6626b0d0c176f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732137393881 2024-11-20T21:16:35,957 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ba04bb2bd029935d159faf0939a1f26#C#compaction#565 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:35,957 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/7f6be9eb54ab408e922cc3e1bfff3efd is 50, key is test_row_0/C:col10/1732137393883/Put/seqid=0 2024-11-20T21:16:35,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742491_1667 (size=12779) 2024-11-20T21:16:36,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-20T21:16:36,227 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:16:36,228 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:16:36,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-20T21:16:36,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T21:16:36,234 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:16:36,234 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:16:36,235 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:16:36,302 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ba04bb2bd029935d159faf0939a1f26#A#compaction#563 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:36,303 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/49235fb7ab6b45ffb676b0c113043c2f is 175, key is test_row_0/A:col10/1732137393883/Put/seqid=0 2024-11-20T21:16:36,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742492_1668 (size=31733) 2024-11-20T21:16:36,325 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/49235fb7ab6b45ffb676b0c113043c2f as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/49235fb7ab6b45ffb676b0c113043c2f 2024-11-20T21:16:36,332 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2ba04bb2bd029935d159faf0939a1f26/A of 2ba04bb2bd029935d159faf0939a1f26 into 49235fb7ab6b45ffb676b0c113043c2f(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:36,332 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:36,332 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., storeName=2ba04bb2bd029935d159faf0939a1f26/A, priority=12, startTime=1732137395841; duration=0sec 2024-11-20T21:16:36,332 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:36,332 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ba04bb2bd029935d159faf0939a1f26:A 2024-11-20T21:16:36,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T21:16:36,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-20T21:16:36,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:36,397 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 2ba04bb2bd029935d159faf0939a1f26 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-20T21:16:36,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=A 2024-11-20T21:16:36,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:36,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=B 2024-11-20T21:16:36,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:36,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=C 2024-11-20T21:16:36,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:36,413 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/7f6be9eb54ab408e922cc3e1bfff3efd as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/7f6be9eb54ab408e922cc3e1bfff3efd 2024-11-20T21:16:36,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205bdd6a1c5d384c01a664b56480b2d2f7_2ba04bb2bd029935d159faf0939a1f26 is 50, key is test_row_0/A:col10/1732137394544/Put/seqid=0 2024-11-20T21:16:36,437 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2ba04bb2bd029935d159faf0939a1f26/C of 2ba04bb2bd029935d159faf0939a1f26 into 7f6be9eb54ab408e922cc3e1bfff3efd(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:36,437 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:36,437 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., storeName=2ba04bb2bd029935d159faf0939a1f26/C, priority=12, startTime=1732137395842; duration=0sec 2024-11-20T21:16:36,437 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:36,437 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ba04bb2bd029935d159faf0939a1f26:C 2024-11-20T21:16:36,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742493_1669 (size=12454) 2024-11-20T21:16:36,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:36,496 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411205bdd6a1c5d384c01a664b56480b2d2f7_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205bdd6a1c5d384c01a664b56480b2d2f7_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:36,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/f5c243587493472981fb591ad9e974e2, store: [table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:36,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/f5c243587493472981fb591ad9e974e2 is 175, key is test_row_0/A:col10/1732137394544/Put/seqid=0 2024-11-20T21:16:36,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742494_1670 (size=31255) 2024-11-20T21:16:36,538 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=483, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/f5c243587493472981fb591ad9e974e2 2024-11-20T21:16:36,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T21:16:36,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/b08c581a326d4b6686ea061fb0c2849d is 50, key is test_row_0/B:col10/1732137394544/Put/seqid=0 2024-11-20T21:16:36,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742495_1671 (size=12301) 2024-11-20T21:16:36,580 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/b08c581a326d4b6686ea061fb0c2849d 2024-11-20T21:16:36,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/02fa7e13fe4643c180f85c657e5182a4 is 50, key is test_row_0/C:col10/1732137394544/Put/seqid=0 2024-11-20T21:16:36,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742496_1672 (size=12301) 2024-11-20T21:16:36,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:36,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:36,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:36,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137456734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:36,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:36,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:36,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:36,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:36,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137456735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:36,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:36,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:36,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:36,739 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:36,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137456736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:36,740 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:36,740 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:36,740 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:36,850 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:36,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137456847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:36,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:36,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:36,850 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:36,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:36,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137456847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:36,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:36,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137456848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:36,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:36,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:36,852 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:36,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:36,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:36,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:36,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T21:16:37,023 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=483 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/02fa7e13fe4643c180f85c657e5182a4 2024-11-20T21:16:37,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/f5c243587493472981fb591ad9e974e2 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/f5c243587493472981fb591ad9e974e2 2024-11-20T21:16:37,045 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/f5c243587493472981fb591ad9e974e2, entries=150, sequenceid=483, filesize=30.5 K 2024-11-20T21:16:37,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/b08c581a326d4b6686ea061fb0c2849d as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/b08c581a326d4b6686ea061fb0c2849d 2024-11-20T21:16:37,053 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/b08c581a326d4b6686ea061fb0c2849d, entries=150, sequenceid=483, filesize=12.0 K 2024-11-20T21:16:37,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/02fa7e13fe4643c180f85c657e5182a4 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/02fa7e13fe4643c180f85c657e5182a4 2024-11-20T21:16:37,058 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/02fa7e13fe4643c180f85c657e5182a4, entries=150, sequenceid=483, filesize=12.0 K 2024-11-20T21:16:37,058 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 2ba04bb2bd029935d159faf0939a1f26 in 661ms, sequenceid=483, compaction requested=false 2024-11-20T21:16:37,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:37,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:37,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-20T21:16:37,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-20T21:16:37,064 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-20T21:16:37,064 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 825 msec 2024-11-20T21:16:37,066 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 836 msec 2024-11-20T21:16:37,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:37,069 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2ba04bb2bd029935d159faf0939a1f26 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T21:16:37,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=A 2024-11-20T21:16:37,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:37,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=B 2024-11-20T21:16:37,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:37,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=C 2024-11-20T21:16:37,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:37,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112006180952b98043159bc92dfda02e4453_2ba04bb2bd029935d159faf0939a1f26 is 50, key is test_row_0/A:col10/1732137397066/Put/seqid=0 2024-11-20T21:16:37,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:37,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137457096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:37,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:37,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:37,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137457098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:37,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:37,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:37,105 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:37,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:37,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137457101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:37,109 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:37,109 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:37,109 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:37,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742497_1673 (size=14994) 2024-11-20T21:16:37,128 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:37,138 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112006180952b98043159bc92dfda02e4453_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112006180952b98043159bc92dfda02e4453_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:37,140 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/21ac254042a34698ab99fe975f2de561, store: [table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:37,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/21ac254042a34698ab99fe975f2de561 is 175, key is test_row_0/A:col10/1732137397066/Put/seqid=0 2024-11-20T21:16:37,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742498_1674 (size=39949) 2024-11-20T21:16:37,189 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=502, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/21ac254042a34698ab99fe975f2de561 2024-11-20T21:16:37,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:37,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137457208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:37,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:37,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:37,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:37,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:37,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137457209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:37,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:37,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:37,212 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:37,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/09b226dba6df4bd08abb7992c37afb4c is 50, key is test_row_0/B:col10/1732137397066/Put/seqid=0 2024-11-20T21:16:37,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:37,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137457216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:37,220 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:37,220 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:37,220 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:37,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742499_1675 (size=12301) 2024-11-20T21:16:37,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/09b226dba6df4bd08abb7992c37afb4c 2024-11-20T21:16:37,250 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/184b788263d24e5e8e73e48e18721812 is 50, key is test_row_0/C:col10/1732137397066/Put/seqid=0 2024-11-20T21:16:37,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742500_1676 (size=12301) 2024-11-20T21:16:37,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-20T21:16:37,367 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:16:37,367 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:16:37,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-20T21:16:37,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T21:16:37,369 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:16:37,369 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:16:37,370 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:16:37,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:37,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137457417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:37,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:37,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:37,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137457417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:37,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:37,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:37,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:37,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:37,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:37,429 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:37,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137457427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:37,429 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:37,429 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:37,429 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:37,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T21:16:37,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-20T21:16:37,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:37,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:37,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:37,526 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:37,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:37,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:37,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=502 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/184b788263d24e5e8e73e48e18721812 2024-11-20T21:16:37,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-20T21:16:37,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:37,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:37,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:37,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:37,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:37,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:37,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T21:16:37,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/21ac254042a34698ab99fe975f2de561 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/21ac254042a34698ab99fe975f2de561 2024-11-20T21:16:37,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/21ac254042a34698ab99fe975f2de561, entries=200, sequenceid=502, filesize=39.0 K 2024-11-20T21:16:37,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/09b226dba6df4bd08abb7992c37afb4c as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/09b226dba6df4bd08abb7992c37afb4c 2024-11-20T21:16:37,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/09b226dba6df4bd08abb7992c37afb4c, entries=150, sequenceid=502, filesize=12.0 K 2024-11-20T21:16:37,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/184b788263d24e5e8e73e48e18721812 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/184b788263d24e5e8e73e48e18721812 2024-11-20T21:16:37,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/184b788263d24e5e8e73e48e18721812, entries=150, sequenceid=502, filesize=12.0 K 2024-11-20T21:16:37,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 2ba04bb2bd029935d159faf0939a1f26 in 643ms, sequenceid=502, compaction requested=true 2024-11-20T21:16:37,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:37,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ba04bb2bd029935d159faf0939a1f26:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:16:37,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:37,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ba04bb2bd029935d159faf0939a1f26:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:16:37,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:37,713 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:16:37,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ba04bb2bd029935d159faf0939a1f26:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:16:37,713 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:16:37,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:16:37,714 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:16:37,714 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 2ba04bb2bd029935d159faf0939a1f26/B is initiating minor compaction (all files) 2024-11-20T21:16:37,715 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ba04bb2bd029935d159faf0939a1f26/B in TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:37,715 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/0ef4640f4d6b4bdabccf244a638d251b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/b08c581a326d4b6686ea061fb0c2849d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/09b226dba6df4bd08abb7992c37afb4c] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp, totalSize=36.5 K 2024-11-20T21:16:37,715 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102937 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:16:37,715 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 2ba04bb2bd029935d159faf0939a1f26/A is initiating minor compaction (all files) 2024-11-20T21:16:37,715 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ba04bb2bd029935d159faf0939a1f26/A in TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:37,715 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/49235fb7ab6b45ffb676b0c113043c2f, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/f5c243587493472981fb591ad9e974e2, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/21ac254042a34698ab99fe975f2de561] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp, totalSize=100.5 K 2024-11-20T21:16:37,715 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:37,715 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. files: [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/49235fb7ab6b45ffb676b0c113043c2f, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/f5c243587493472981fb591ad9e974e2, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/21ac254042a34698ab99fe975f2de561] 2024-11-20T21:16:37,716 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ef4640f4d6b4bdabccf244a638d251b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732137393881 2024-11-20T21:16:37,716 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49235fb7ab6b45ffb676b0c113043c2f, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732137393881 2024-11-20T21:16:37,716 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting b08c581a326d4b6686ea061fb0c2849d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1732137394535 2024-11-20T21:16:37,716 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5c243587493472981fb591ad9e974e2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1732137394535 2024-11-20T21:16:37,717 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21ac254042a34698ab99fe975f2de561, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1732137396727 2024-11-20T21:16:37,717 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 09b226dba6df4bd08abb7992c37afb4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1732137396734 2024-11-20T21:16:37,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:37,730 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2ba04bb2bd029935d159faf0939a1f26 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-20T21:16:37,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=A 2024-11-20T21:16:37,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:37,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=B 2024-11-20T21:16:37,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:37,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=C 2024-11-20T21:16:37,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:37,733 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ba04bb2bd029935d159faf0939a1f26#B#compaction#572 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:37,734 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/b3e681715c1345babf4a8ccfcdd5c331 is 50, key is test_row_0/B:col10/1732137397066/Put/seqid=0 2024-11-20T21:16:37,737 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:37,745 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112092353f9dffb541ff917ce53944323e27_2ba04bb2bd029935d159faf0939a1f26 store=[table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:37,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742501_1677 (size=12881) 2024-11-20T21:16:37,747 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112092353f9dffb541ff917ce53944323e27_2ba04bb2bd029935d159faf0939a1f26, store=[table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:37,747 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112092353f9dffb541ff917ce53944323e27_2ba04bb2bd029935d159faf0939a1f26 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:37,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200d83d20b0c98420d9e8fc19cc0d53c87_2ba04bb2bd029935d159faf0939a1f26 is 50, key is test_row_0/A:col10/1732137397091/Put/seqid=0 2024-11-20T21:16:37,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742503_1679 (size=14994) 2024-11-20T21:16:37,751 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:37,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742502_1678 (size=4469) 2024-11-20T21:16:37,752 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/b3e681715c1345babf4a8ccfcdd5c331 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/b3e681715c1345babf4a8ccfcdd5c331 2024-11-20T21:16:37,755 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200d83d20b0c98420d9e8fc19cc0d53c87_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200d83d20b0c98420d9e8fc19cc0d53c87_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:37,756 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/48f1b09d501346faae45e3e8c1d64db6, store: [table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:37,757 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2ba04bb2bd029935d159faf0939a1f26/B of 2ba04bb2bd029935d159faf0939a1f26 into b3e681715c1345babf4a8ccfcdd5c331(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:37,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/48f1b09d501346faae45e3e8c1d64db6 is 175, key is test_row_0/A:col10/1732137397091/Put/seqid=0 2024-11-20T21:16:37,757 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:37,757 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., storeName=2ba04bb2bd029935d159faf0939a1f26/B, priority=13, startTime=1732137397713; duration=0sec 2024-11-20T21:16:37,757 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:16:37,757 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ba04bb2bd029935d159faf0939a1f26:B 2024-11-20T21:16:37,757 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-20T21:16:37,758 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-20T21:16:37,758 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 2ba04bb2bd029935d159faf0939a1f26/C is initiating minor compaction (all files) 2024-11-20T21:16:37,758 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ba04bb2bd029935d159faf0939a1f26/C in TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:37,759 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/7f6be9eb54ab408e922cc3e1bfff3efd, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/02fa7e13fe4643c180f85c657e5182a4, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/184b788263d24e5e8e73e48e18721812] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp, totalSize=36.5 K 2024-11-20T21:16:37,759 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f6be9eb54ab408e922cc3e1bfff3efd, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=465, earliestPutTs=1732137393881 2024-11-20T21:16:37,759 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 02fa7e13fe4643c180f85c657e5182a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=483, earliestPutTs=1732137394535 2024-11-20T21:16:37,759 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 184b788263d24e5e8e73e48e18721812, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1732137396734 2024-11-20T21:16:37,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742504_1680 (size=39949) 2024-11-20T21:16:37,761 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=517, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/48f1b09d501346faae45e3e8c1d64db6 2024-11-20T21:16:37,765 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ba04bb2bd029935d159faf0939a1f26#C#compaction#575 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:37,766 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/2543955b4f6a410cada7713e99d60aa3 is 50, key is test_row_0/C:col10/1732137397066/Put/seqid=0 2024-11-20T21:16:37,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:37,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137457759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:37,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:37,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:37,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:37,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/2cb3c1420a274bd384472aff47b86a4d is 50, key is test_row_0/B:col10/1732137397091/Put/seqid=0 2024-11-20T21:16:37,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742505_1681 (size=12881) 2024-11-20T21:16:37,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:37,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137457763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:37,769 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:37,769 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:37,769 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:37,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742506_1682 (size=12301) 2024-11-20T21:16:37,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:37,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137457766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:37,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:37,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:37,772 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:37,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-20T21:16:37,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:37,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:37,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:37,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:37,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:37,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:37,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:37,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137457877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:37,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:37,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137457877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:37,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:37,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:37,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:37,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:37,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:37,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:37,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:37,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137457877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:37,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:37,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:37,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:37,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-20T21:16:37,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:37,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:37,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:37,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:37,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:37,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T21:16:38,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:38,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137458086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:38,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:38,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:38,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:38,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:38,091 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:38,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137458086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:38,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137458087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:38,092 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:38,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:38,092 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:38,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:38,092 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:38,092 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:38,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-20T21:16:38,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:38,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:38,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:38,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:38,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:38,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:38,157 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ba04bb2bd029935d159faf0939a1f26#A#compaction#573 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:38,158 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/8c8fddba05324742a2577a8a5f2b7dde is 175, key is test_row_0/A:col10/1732137397066/Put/seqid=0 2024-11-20T21:16:38,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742507_1683 (size=31835) 2024-11-20T21:16:38,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/2cb3c1420a274bd384472aff47b86a4d 2024-11-20T21:16:38,177 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/8c8fddba05324742a2577a8a5f2b7dde as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/8c8fddba05324742a2577a8a5f2b7dde 2024-11-20T21:16:38,178 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/2543955b4f6a410cada7713e99d60aa3 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/2543955b4f6a410cada7713e99d60aa3 2024-11-20T21:16:38,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/c3fcd37b25d249628527fe41e8324d31 is 50, key is test_row_0/C:col10/1732137397091/Put/seqid=0 2024-11-20T21:16:38,187 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2ba04bb2bd029935d159faf0939a1f26/A of 2ba04bb2bd029935d159faf0939a1f26 into 8c8fddba05324742a2577a8a5f2b7dde(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:38,187 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2ba04bb2bd029935d159faf0939a1f26/C of 2ba04bb2bd029935d159faf0939a1f26 into 2543955b4f6a410cada7713e99d60aa3(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:38,187 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:38,187 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:38,187 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., storeName=2ba04bb2bd029935d159faf0939a1f26/A, priority=13, startTime=1732137397713; duration=0sec 2024-11-20T21:16:38,187 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., storeName=2ba04bb2bd029935d159faf0939a1f26/C, priority=13, startTime=1732137397713; duration=0sec 2024-11-20T21:16:38,187 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:38,187 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ba04bb2bd029935d159faf0939a1f26:A 2024-11-20T21:16:38,187 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:38,187 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ba04bb2bd029935d159faf0939a1f26:C 2024-11-20T21:16:38,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742508_1684 (size=12301) 2024-11-20T21:16:38,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/c3fcd37b25d249628527fe41e8324d31 2024-11-20T21:16:38,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/48f1b09d501346faae45e3e8c1d64db6 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/48f1b09d501346faae45e3e8c1d64db6 2024-11-20T21:16:38,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/48f1b09d501346faae45e3e8c1d64db6, entries=200, sequenceid=517, filesize=39.0 K 2024-11-20T21:16:38,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/2cb3c1420a274bd384472aff47b86a4d as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/2cb3c1420a274bd384472aff47b86a4d 2024-11-20T21:16:38,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/2cb3c1420a274bd384472aff47b86a4d, entries=150, sequenceid=517, filesize=12.0 K 2024-11-20T21:16:38,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/c3fcd37b25d249628527fe41e8324d31 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/c3fcd37b25d249628527fe41e8324d31 2024-11-20T21:16:38,238 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/c3fcd37b25d249628527fe41e8324d31, entries=150, sequenceid=517, filesize=12.0 K 2024-11-20T21:16:38,240 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 2ba04bb2bd029935d159faf0939a1f26 in 509ms, sequenceid=517, compaction requested=false 2024-11-20T21:16:38,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:38,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-20T21:16:38,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:38,297 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 2ba04bb2bd029935d159faf0939a1f26 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-20T21:16:38,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=A 2024-11-20T21:16:38,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:38,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=B 2024-11-20T21:16:38,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:38,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=C 2024-11-20T21:16:38,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:38,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e4cd6d5dad4847e9afdaceccc0bcb5b4_2ba04bb2bd029935d159faf0939a1f26 is 50, key is test_row_0/A:col10/1732137397764/Put/seqid=0 2024-11-20T21:16:38,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742509_1685 (size=12454) 2024-11-20T21:16:38,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:38,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:38,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:38,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137458418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:38,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:38,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:38,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:38,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137458419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:38,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:38,424 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:38,424 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:38,424 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:38,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:38,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137458420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:38,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:38,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:38,425 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:38,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T21:16:38,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:38,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137458527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:38,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:38,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137458527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:38,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:38,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:38,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:38,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:38,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137458527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:38,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:38,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:38,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:38,532 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:38,532 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:38,533 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:38,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137458737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:38,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:38,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137458738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:38,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:38,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:38,741 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:38,741 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:38,741 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:38,741 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:38,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137458738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:38,742 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:38,742 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:38,742 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:38,762 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e4cd6d5dad4847e9afdaceccc0bcb5b4_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e4cd6d5dad4847e9afdaceccc0bcb5b4_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:38,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/c2ee6466447444f98888c7631ca7ff37, store: [table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:38,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/c2ee6466447444f98888c7631ca7ff37 is 175, key is test_row_0/A:col10/1732137397764/Put/seqid=0 2024-11-20T21:16:38,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742510_1686 (size=31255) 2024-11-20T21:16:39,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:39,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137459046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:39,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:39,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:39,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137459047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:39,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:39,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:39,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:39,051 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:39,051 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:39,051 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:39,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137459047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:39,051 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:39,051 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:39,051 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:39,175 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=537, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/c2ee6466447444f98888c7631ca7ff37 2024-11-20T21:16:39,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/5068b4cced164f5082fb70e0fbbaf2ca is 50, key is test_row_0/B:col10/1732137397764/Put/seqid=0 2024-11-20T21:16:39,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742511_1687 (size=12301) 2024-11-20T21:16:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T21:16:39,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:39,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137459561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:39,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:39,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:39,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:39,564 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:39,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137459561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:39,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:39,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:39,565 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:39,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:39,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137459561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:39,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:39,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:39,569 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:39,595 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/5068b4cced164f5082fb70e0fbbaf2ca 2024-11-20T21:16:39,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/c13f24c83eda48c0b28db51190f99d46 is 50, key is test_row_0/C:col10/1732137397764/Put/seqid=0 2024-11-20T21:16:39,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742512_1688 (size=12301) 2024-11-20T21:16:40,022 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=537 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/c13f24c83eda48c0b28db51190f99d46 2024-11-20T21:16:40,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/c2ee6466447444f98888c7631ca7ff37 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/c2ee6466447444f98888c7631ca7ff37 2024-11-20T21:16:40,039 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/c2ee6466447444f98888c7631ca7ff37, entries=150, sequenceid=537, filesize=30.5 K 2024-11-20T21:16:40,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/5068b4cced164f5082fb70e0fbbaf2ca as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/5068b4cced164f5082fb70e0fbbaf2ca 2024-11-20T21:16:40,045 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/5068b4cced164f5082fb70e0fbbaf2ca, entries=150, sequenceid=537, filesize=12.0 K 2024-11-20T21:16:40,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/c13f24c83eda48c0b28db51190f99d46 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/c13f24c83eda48c0b28db51190f99d46 2024-11-20T21:16:40,073 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/c13f24c83eda48c0b28db51190f99d46, entries=150, sequenceid=537, filesize=12.0 K 2024-11-20T21:16:40,081 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 2ba04bb2bd029935d159faf0939a1f26 in 1784ms, sequenceid=537, compaction requested=true 2024-11-20T21:16:40,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:40,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:40,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-20T21:16:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-20T21:16:40,107 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-20T21:16:40,107 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7220 sec 2024-11-20T21:16:40,113 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 2.7420 sec 2024-11-20T21:16:40,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2ba04bb2bd029935d159faf0939a1f26 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-20T21:16:40,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=A 2024-11-20T21:16:40,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:40,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=B 2024-11-20T21:16:40,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:40,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=C 2024-11-20T21:16:40,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:40,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:40,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120002b58a43bdb4756a66a65f880d6618e_2ba04bb2bd029935d159faf0939a1f26 is 50, key is test_row_0/A:col10/1732137400583/Put/seqid=0 2024-11-20T21:16:40,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742513_1689 (size=12454) 2024-11-20T21:16:40,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:40,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137460679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:40,689 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:40,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:40,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137460681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:40,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:40,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:40,690 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:40,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:40,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137460688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:40,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:40,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:40,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:40,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:40,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137460797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:40,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:40,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137460797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:40,803 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:40,803 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:40,803 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:40,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:40,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:40,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:40,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137460798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:40,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:40,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:40,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:40,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:41,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:41,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137461007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:41,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:41,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137461007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:41,010 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:41,010 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:41,010 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:41,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:41,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:41,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:41,010 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:41,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137461007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:41,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:41,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:41,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:41,025 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:41,028 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120002b58a43bdb4756a66a65f880d6618e_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120002b58a43bdb4756a66a65f880d6618e_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:41,029 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/3cc9626b5a72494485e9fc3c0f7119e1, store: [table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:41,030 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/3cc9626b5a72494485e9fc3c0f7119e1 is 175, key is test_row_0/A:col10/1732137400583/Put/seqid=0 2024-11-20T21:16:41,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742514_1690 (size=31255) 2024-11-20T21:16:41,033 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=555, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/3cc9626b5a72494485e9fc3c0f7119e1 2024-11-20T21:16:41,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/c0be749bc380496c87bd6489e044cf64 is 50, key is test_row_0/B:col10/1732137400583/Put/seqid=0 2024-11-20T21:16:41,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742515_1691 (size=12301) 2024-11-20T21:16:41,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:41,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137461316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:41,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:41,321 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:41,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137461317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:41,321 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:41,321 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:41,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:41,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137461317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:41,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:41,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:41,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:41,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:41,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:41,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:41,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/c0be749bc380496c87bd6489e044cf64 2024-11-20T21:16:41,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/55a91c0068be41c2bc7332e7d1ae4573 is 50, key is test_row_0/C:col10/1732137400583/Put/seqid=0 2024-11-20T21:16:41,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742516_1692 (size=12301) 2024-11-20T21:16:41,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-20T21:16:41,527 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:16:41,528 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-20T21:16:41,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-20T21:16:41,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-20T21:16:41,530 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-20T21:16:41,531 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-20T21:16:41,531 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-20T21:16:41,580 INFO [Thread-2686 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2686. 2024-11-20T21:16:41,580 DEBUG [Thread-2686 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:16:41,580 DEBUG [Thread-2686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:41,580 DEBUG [Thread-2686 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:41,580 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:16:41,583 INFO [Thread-2690 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2690. 2024-11-20T21:16:41,583 DEBUG [Thread-2690 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:16:41,583 DEBUG [Thread-2690 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:41,584 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:16:41,584 DEBUG [Thread-2690 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:41,585 INFO [Thread-2688 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2688. 2024-11-20T21:16:41,585 DEBUG [Thread-2688 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:16:41,585 DEBUG [Thread-2688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:41,585 DEBUG [Thread-2688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:41,585 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:16:41,586 INFO [Thread-2684 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2684. 2024-11-20T21:16:41,586 DEBUG [Thread-2684 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:16:41,586 DEBUG [Thread-2684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:41,586 DEBUG [Thread-2684 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:41,586 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:16:41,587 INFO [Thread-2692 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2692. 2024-11-20T21:16:41,587 DEBUG [Thread-2692 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:16:41,587 DEBUG [Thread-2692 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:41,587 DEBUG [Thread-2692 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:41,587 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:16:41,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-20T21:16:41,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-20T21:16:41,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:41,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:41,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:41,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:41,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:41,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:41,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:41,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:41,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T21:16:41,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44250 deadline: 1732137461827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:41,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44316 deadline: 1732137461827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:41,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:44280 deadline: 1732137461827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:41,827 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:41,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:41,827 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:41,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 , the old value is region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:41,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:41,827 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:41,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-20T21:16:41,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:41,828 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., hostname=0b16ab3f13b7,36987,1732137236536, seqNum=5 because the exception is null or not the one we care about 2024-11-20T21:16:41,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-20T21:16:41,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:41,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:41,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:41,836 ERROR [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:41,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:41,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T21:16:41,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-20T21:16:41,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/55a91c0068be41c2bc7332e7d1ae4573 2024-11-20T21:16:41,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/3cc9626b5a72494485e9fc3c0f7119e1 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/3cc9626b5a72494485e9fc3c0f7119e1 2024-11-20T21:16:41,867 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/3cc9626b5a72494485e9fc3c0f7119e1, entries=150, sequenceid=555, filesize=30.5 K 2024-11-20T21:16:41,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/c0be749bc380496c87bd6489e044cf64 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/c0be749bc380496c87bd6489e044cf64 2024-11-20T21:16:41,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/c0be749bc380496c87bd6489e044cf64, entries=150, sequenceid=555, filesize=12.0 K 2024-11-20T21:16:41,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/55a91c0068be41c2bc7332e7d1ae4573 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/55a91c0068be41c2bc7332e7d1ae4573 2024-11-20T21:16:41,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/55a91c0068be41c2bc7332e7d1ae4573, entries=150, sequenceid=555, filesize=12.0 K 2024-11-20T21:16:41,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 2ba04bb2bd029935d159faf0939a1f26 in 1279ms, sequenceid=555, compaction requested=true 2024-11-20T21:16:41,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:41,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ba04bb2bd029935d159faf0939a1f26:A, priority=-2147483648, current under compaction store size is 1 2024-11-20T21:16:41,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:41,873 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:16:41,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ba04bb2bd029935d159faf0939a1f26:B, priority=-2147483648, current under compaction store size is 2 2024-11-20T21:16:41,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:41,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2ba04bb2bd029935d159faf0939a1f26:C, priority=-2147483648, current under compaction store size is 3 2024-11-20T21:16:41,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:16:41,873 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:16:41,873 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134294 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:16:41,873 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:16:41,873 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 2ba04bb2bd029935d159faf0939a1f26/B is initiating minor compaction (all files) 2024-11-20T21:16:41,873 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1541): 2ba04bb2bd029935d159faf0939a1f26/A is initiating minor compaction (all files) 2024-11-20T21:16:41,873 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ba04bb2bd029935d159faf0939a1f26/A in TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:41,873 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ba04bb2bd029935d159faf0939a1f26/B in TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:41,874 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/8c8fddba05324742a2577a8a5f2b7dde, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/48f1b09d501346faae45e3e8c1d64db6, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/c2ee6466447444f98888c7631ca7ff37, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/3cc9626b5a72494485e9fc3c0f7119e1] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp, totalSize=131.1 K 2024-11-20T21:16:41,874 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/b3e681715c1345babf4a8ccfcdd5c331, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/2cb3c1420a274bd384472aff47b86a4d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/5068b4cced164f5082fb70e0fbbaf2ca, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/c0be749bc380496c87bd6489e044cf64] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp, totalSize=48.6 K 2024-11-20T21:16:41,874 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:41,874 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. files: [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/8c8fddba05324742a2577a8a5f2b7dde, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/48f1b09d501346faae45e3e8c1d64db6, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/c2ee6466447444f98888c7631ca7ff37, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/3cc9626b5a72494485e9fc3c0f7119e1] 2024-11-20T21:16:41,874 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting b3e681715c1345babf4a8ccfcdd5c331, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1732137396734 2024-11-20T21:16:41,874 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c8fddba05324742a2577a8a5f2b7dde, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1732137396734 2024-11-20T21:16:41,874 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cb3c1420a274bd384472aff47b86a4d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=517, earliestPutTs=1732137397091 2024-11-20T21:16:41,874 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48f1b09d501346faae45e3e8c1d64db6, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=517, earliestPutTs=1732137397091 2024-11-20T21:16:41,874 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 5068b4cced164f5082fb70e0fbbaf2ca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1732137397750 2024-11-20T21:16:41,874 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2ee6466447444f98888c7631ca7ff37, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1732137397750 2024-11-20T21:16:41,874 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting c0be749bc380496c87bd6489e044cf64, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1732137398416 2024-11-20T21:16:41,874 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cc9626b5a72494485e9fc3c0f7119e1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1732137398416 2024-11-20T21:16:41,879 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:41,880 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ba04bb2bd029935d159faf0939a1f26#B#compaction#584 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:41,880 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241120e16950ce4c9e488e93a9199882f083ad_2ba04bb2bd029935d159faf0939a1f26 store=[table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:41,880 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/3f232878f17f4709b1ad7c098fe23b03 is 50, key is test_row_0/B:col10/1732137400583/Put/seqid=0 2024-11-20T21:16:41,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742517_1693 (size=13017) 2024-11-20T21:16:41,885 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241120e16950ce4c9e488e93a9199882f083ad_2ba04bb2bd029935d159faf0939a1f26, store=[table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:41,885 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241120e16950ce4c9e488e93a9199882f083ad_2ba04bb2bd029935d159faf0939a1f26 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:41,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742518_1694 (size=4469) 2024-11-20T21:16:41,889 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ba04bb2bd029935d159faf0939a1f26#A#compaction#585 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:41,889 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/3f232878f17f4709b1ad7c098fe23b03 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/3f232878f17f4709b1ad7c098fe23b03 2024-11-20T21:16:41,889 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/8ed93d5118564170b2ec9cda5d7eda0c is 175, key is test_row_0/A:col10/1732137400583/Put/seqid=0 2024-11-20T21:16:41,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742519_1695 (size=31971) 2024-11-20T21:16:41,893 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2ba04bb2bd029935d159faf0939a1f26/B of 2ba04bb2bd029935d159faf0939a1f26 into 3f232878f17f4709b1ad7c098fe23b03(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:41,893 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:41,893 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., storeName=2ba04bb2bd029935d159faf0939a1f26/B, priority=12, startTime=1732137401873; duration=0sec 2024-11-20T21:16:41,893 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-20T21:16:41,893 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ba04bb2bd029935d159faf0939a1f26:B 2024-11-20T21:16:41,893 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-20T21:16:41,894 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-20T21:16:41,894 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1541): 2ba04bb2bd029935d159faf0939a1f26/C is initiating minor compaction (all files) 2024-11-20T21:16:41,894 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2ba04bb2bd029935d159faf0939a1f26/C in TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:41,894 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/2543955b4f6a410cada7713e99d60aa3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/c3fcd37b25d249628527fe41e8324d31, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/c13f24c83eda48c0b28db51190f99d46, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/55a91c0068be41c2bc7332e7d1ae4573] into tmpdir=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp, totalSize=48.6 K 2024-11-20T21:16:41,894 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 2543955b4f6a410cada7713e99d60aa3, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=502, earliestPutTs=1732137396734 2024-11-20T21:16:41,895 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting c3fcd37b25d249628527fe41e8324d31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=517, earliestPutTs=1732137397091 2024-11-20T21:16:41,895 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting c13f24c83eda48c0b28db51190f99d46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=537, earliestPutTs=1732137397750 2024-11-20T21:16:41,895 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] compactions.Compactor(225): Compacting 55a91c0068be41c2bc7332e7d1ae4573, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=555, earliestPutTs=1732137398416 2024-11-20T21:16:41,896 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/8ed93d5118564170b2ec9cda5d7eda0c as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/8ed93d5118564170b2ec9cda5d7eda0c 2024-11-20T21:16:41,900 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2ba04bb2bd029935d159faf0939a1f26/A of 2ba04bb2bd029935d159faf0939a1f26 into 8ed93d5118564170b2ec9cda5d7eda0c(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:41,900 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:41,900 INFO [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., storeName=2ba04bb2bd029935d159faf0939a1f26/A, priority=12, startTime=1732137401872; duration=0sec 2024-11-20T21:16:41,900 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:41,900 DEBUG [RS:0;0b16ab3f13b7:36987-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ba04bb2bd029935d159faf0939a1f26:A 2024-11-20T21:16:41,902 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2ba04bb2bd029935d159faf0939a1f26#C#compaction#586 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-20T21:16:41,902 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/a5085a556ca9420ab62e482748fd1781 is 50, key is test_row_0/C:col10/1732137400583/Put/seqid=0 2024-11-20T21:16:41,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742520_1696 (size=13017) 2024-11-20T21:16:41,907 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/a5085a556ca9420ab62e482748fd1781 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/a5085a556ca9420ab62e482748fd1781 2024-11-20T21:16:41,910 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2ba04bb2bd029935d159faf0939a1f26/C of 2ba04bb2bd029935d159faf0939a1f26 into a5085a556ca9420ab62e482748fd1781(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-20T21:16:41,910 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:41,910 INFO [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., storeName=2ba04bb2bd029935d159faf0939a1f26/C, priority=12, startTime=1732137401873; duration=0sec 2024-11-20T21:16:41,910 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-20T21:16:41,910 DEBUG [RS:0;0b16ab3f13b7:36987-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2ba04bb2bd029935d159faf0939a1f26:C 2024-11-20T21:16:41,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36987 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-20T21:16:41,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:41,989 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 2ba04bb2bd029935d159faf0939a1f26 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-20T21:16:41,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=A 2024-11-20T21:16:41,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:41,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=B 2024-11-20T21:16:41,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:41,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=C 2024-11-20T21:16:41,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:41,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200e8f9b13192142e8a6ccd654a93a5d26_2ba04bb2bd029935d159faf0939a1f26 is 50, key is test_row_0/A:col10/1732137400686/Put/seqid=0 2024-11-20T21:16:41,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742521_1697 (size=12454) 2024-11-20T21:16:42,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-20T21:16:42,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. as already flushing 2024-11-20T21:16:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36987 {}] regionserver.HRegion(8855): Flush requested on 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:42,188 INFO [Thread-2673 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2673. 2024-11-20T21:16:42,188 DEBUG [Thread-2673 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:16:42,188 DEBUG [Thread-2673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:42,188 DEBUG [Thread-2673 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:42,188 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:16:42,228 INFO [Thread-2675 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2675. 2024-11-20T21:16:42,228 DEBUG [Thread-2675 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:16:42,228 DEBUG [Thread-2675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:42,228 DEBUG [Thread-2675 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:42,228 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:16:42,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:42,400 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411200e8f9b13192142e8a6ccd654a93a5d26_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200e8f9b13192142e8a6ccd654a93a5d26_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:42,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/8b0ab4f247a3485aa819734aa52ea103, store: [table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:42,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/8b0ab4f247a3485aa819734aa52ea103 is 175, key is test_row_0/A:col10/1732137400686/Put/seqid=0 2024-11-20T21:16:42,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742522_1698 (size=31255) 2024-11-20T21:16:42,405 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=572, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/8b0ab4f247a3485aa819734aa52ea103 2024-11-20T21:16:42,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/baa6ab2db1c54ba5b9d10333d43e1eee is 50, key is test_row_0/B:col10/1732137400686/Put/seqid=0 2024-11-20T21:16:42,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742523_1699 (size=12301) 2024-11-20T21:16:42,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-20T21:16:42,814 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/baa6ab2db1c54ba5b9d10333d43e1eee 2024-11-20T21:16:42,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/35c763ea9f7244d1acffc8902ce371c7 is 50, key is test_row_0/C:col10/1732137400686/Put/seqid=0 2024-11-20T21:16:42,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742524_1700 (size=12301) 2024-11-20T21:16:42,838 INFO [Thread-2679 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2679. 2024-11-20T21:16:42,839 DEBUG [Thread-2679 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:16:42,839 DEBUG [Thread-2679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:42,839 DEBUG [Thread-2679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:42,839 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:16:42,848 INFO [Thread-2677 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2677. 2024-11-20T21:16:42,848 DEBUG [Thread-2677 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:16:42,848 INFO [Thread-2681 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2681. 2024-11-20T21:16:42,848 DEBUG [Thread-2677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:42,848 DEBUG [Thread-2681 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-20T21:16:42,848 DEBUG [Thread-2681 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:42,848 DEBUG [Thread-2677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:42,848 DEBUG [Thread-2681 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:42,848 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:16:42,848 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:16:43,225 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=572 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/35c763ea9f7244d1acffc8902ce371c7 2024-11-20T21:16:43,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/8b0ab4f247a3485aa819734aa52ea103 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/8b0ab4f247a3485aa819734aa52ea103 2024-11-20T21:16:43,232 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/8b0ab4f247a3485aa819734aa52ea103, entries=150, sequenceid=572, filesize=30.5 K 2024-11-20T21:16:43,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/baa6ab2db1c54ba5b9d10333d43e1eee as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/baa6ab2db1c54ba5b9d10333d43e1eee 2024-11-20T21:16:43,235 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/baa6ab2db1c54ba5b9d10333d43e1eee, entries=150, sequenceid=572, filesize=12.0 K 2024-11-20T21:16:43,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/35c763ea9f7244d1acffc8902ce371c7 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/35c763ea9f7244d1acffc8902ce371c7 2024-11-20T21:16:43,238 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/35c763ea9f7244d1acffc8902ce371c7, entries=150, sequenceid=572, filesize=12.0 K 2024-11-20T21:16:43,238 INFO [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=33.54 KB/34350 for 2ba04bb2bd029935d159faf0939a1f26 in 1250ms, sequenceid=572, compaction requested=false 2024-11-20T21:16:43,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 2ba04bb2bd029935d159faf0939a1f26: 2024-11-20T21:16:43,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:43,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/0b16ab3f13b7:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-20T21:16:43,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-20T21:16:43,241 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-20T21:16:43,241 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7090 sec 2024-11-20T21:16:43,242 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.7130 sec 2024-11-20T21:16:43,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-20T21:16:43,677 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:16:43,677 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-20T21:16:43,677 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-20T21:16:43,677 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-20T21:16:43,677 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 154 2024-11-20T21:16:43,677 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 161 2024-11-20T21:16:43,677 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 152 2024-11-20T21:16:43,677 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-20T21:16:43,677 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2340 2024-11-20T21:16:43,677 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2416 2024-11-20T21:16:43,677 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2442 2024-11-20T21:16:43,677 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2347 2024-11-20T21:16:43,677 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2403 2024-11-20T21:16:43,677 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-20T21:16:43,677 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T21:16:43,677 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T21:16:43,677 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:43,678 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:43,678 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:16:43,678 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-20T21:16:43,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-20T21:16:43,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-20T21:16:43,681 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732137403681"}]},"ts":"1732137403681"} 2024-11-20T21:16:43,682 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-20T21:16:43,682 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-20T21:16:43,683 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-20T21:16:43,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2ba04bb2bd029935d159faf0939a1f26, UNASSIGN}] 2024-11-20T21:16:43,685 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2ba04bb2bd029935d159faf0939a1f26, UNASSIGN 2024-11-20T21:16:43,686 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=2ba04bb2bd029935d159faf0939a1f26, regionState=CLOSING, regionLocation=0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:43,687 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=168, ppid=167, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2ba04bb2bd029935d159faf0939a1f26, UNASSIGN because future has completed 2024-11-20T21:16:43,687 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T21:16:43,687 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536}] 2024-11-20T21:16:43,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-20T21:16:43,839 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(122): Close 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:43,840 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T21:16:43,840 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1722): Closing 2ba04bb2bd029935d159faf0939a1f26, disabling compactions & flushes 2024-11-20T21:16:43,840 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:43,840 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:43,840 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. after waiting 0 ms 2024-11-20T21:16:43,840 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:43,840 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(2902): Flushing 2ba04bb2bd029935d159faf0939a1f26 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-20T21:16:43,840 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=A 2024-11-20T21:16:43,840 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:43,840 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=B 2024-11-20T21:16:43,840 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:43,840 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2ba04bb2bd029935d159faf0939a1f26, store=C 2024-11-20T21:16:43,840 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-20T21:16:43,847 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112016b26f810c8b443bb33bccc0fa89884a_2ba04bb2bd029935d159faf0939a1f26 is 50, key is test_row_0/A:col10/1732137402186/Put/seqid=0 2024-11-20T21:16:43,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742525_1701 (size=12454) 2024-11-20T21:16:43,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-20T21:16:44,251 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T21:16:44,254 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112016b26f810c8b443bb33bccc0fa89884a_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112016b26f810c8b443bb33bccc0fa89884a_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:44,255 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/2eb3a2ee7d4944e9aeaf95d1b0281404, store: [table=TestAcidGuarantees family=A region=2ba04bb2bd029935d159faf0939a1f26] 2024-11-20T21:16:44,255 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/2eb3a2ee7d4944e9aeaf95d1b0281404 is 175, key is test_row_0/A:col10/1732137402186/Put/seqid=0 2024-11-20T21:16:44,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742526_1702 (size=31255) 2024-11-20T21:16:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-20T21:16:44,659 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=580, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/2eb3a2ee7d4944e9aeaf95d1b0281404 2024-11-20T21:16:44,664 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/1f19232491654e76b1416e46dd637ebe is 50, key is test_row_0/B:col10/1732137402186/Put/seqid=0 2024-11-20T21:16:44,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742527_1703 (size=12301) 2024-11-20T21:16:44,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-20T21:16:45,067 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=580 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/1f19232491654e76b1416e46dd637ebe 2024-11-20T21:16:45,073 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/753f232b8d4d46eb84fe4e5615493184 is 50, key is test_row_0/C:col10/1732137402186/Put/seqid=0 2024-11-20T21:16:45,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742528_1704 (size=12301) 2024-11-20T21:16:45,478 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=580 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/753f232b8d4d46eb84fe4e5615493184 2024-11-20T21:16:45,481 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/A/2eb3a2ee7d4944e9aeaf95d1b0281404 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/2eb3a2ee7d4944e9aeaf95d1b0281404 2024-11-20T21:16:45,484 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/2eb3a2ee7d4944e9aeaf95d1b0281404, entries=150, sequenceid=580, filesize=30.5 K 2024-11-20T21:16:45,484 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/B/1f19232491654e76b1416e46dd637ebe as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/1f19232491654e76b1416e46dd637ebe 2024-11-20T21:16:45,486 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/1f19232491654e76b1416e46dd637ebe, entries=150, sequenceid=580, filesize=12.0 K 2024-11-20T21:16:45,487 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/.tmp/C/753f232b8d4d46eb84fe4e5615493184 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/753f232b8d4d46eb84fe4e5615493184 2024-11-20T21:16:45,489 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/753f232b8d4d46eb84fe4e5615493184, entries=150, sequenceid=580, filesize=12.0 K 2024-11-20T21:16:45,490 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 2ba04bb2bd029935d159faf0939a1f26 in 1650ms, sequenceid=580, compaction requested=true 2024-11-20T21:16:45,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/dfe4d180a85d4779a6f70ca1852f2a0b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/c8e889f9273c4c068a701c9da553354b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/9015cc75a4e342129b755478c3186bfd, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/2cfcdbed74e84ffdb9899c50f71615cc, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/e675376333e14aa49750696d0113221c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/6d4ea3e16828498ba44331817bfd978b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/e61a48eadc8f4e85be992a048b132481, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/dc7840f137664093bf5a58bd29c8a468, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/b937926ece0f489a9f054e9729f2662a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/4150d4c5cafd4bd197f5a311bec8bb5d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/40c0355cfba146dd83a6b02e4543fd0d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/f24bb7ced89a4d1092599fe63ca1dcbd, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/4e79ec6cee8e474cae62887eae16aa48, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/49235fb7ab6b45ffb676b0c113043c2f, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/2670f1dcd243477aa8fb124b75157d40, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/f5c243587493472981fb591ad9e974e2, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/21ac254042a34698ab99fe975f2de561, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/8c8fddba05324742a2577a8a5f2b7dde, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/48f1b09d501346faae45e3e8c1d64db6, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/c2ee6466447444f98888c7631ca7ff37, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/3cc9626b5a72494485e9fc3c0f7119e1] to archive 2024-11-20T21:16:45,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T21:16:45,492 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/dfe4d180a85d4779a6f70ca1852f2a0b to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/dfe4d180a85d4779a6f70ca1852f2a0b 2024-11-20T21:16:45,493 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/c8e889f9273c4c068a701c9da553354b to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/c8e889f9273c4c068a701c9da553354b 2024-11-20T21:16:45,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/9015cc75a4e342129b755478c3186bfd to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/9015cc75a4e342129b755478c3186bfd 2024-11-20T21:16:45,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/2cfcdbed74e84ffdb9899c50f71615cc to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/2cfcdbed74e84ffdb9899c50f71615cc 2024-11-20T21:16:45,495 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/e675376333e14aa49750696d0113221c to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/e675376333e14aa49750696d0113221c 2024-11-20T21:16:45,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/6d4ea3e16828498ba44331817bfd978b to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/6d4ea3e16828498ba44331817bfd978b 2024-11-20T21:16:45,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/e61a48eadc8f4e85be992a048b132481 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/e61a48eadc8f4e85be992a048b132481 2024-11-20T21:16:45,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/dc7840f137664093bf5a58bd29c8a468 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/dc7840f137664093bf5a58bd29c8a468 2024-11-20T21:16:45,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/b937926ece0f489a9f054e9729f2662a to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/b937926ece0f489a9f054e9729f2662a 2024-11-20T21:16:45,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/4150d4c5cafd4bd197f5a311bec8bb5d to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/4150d4c5cafd4bd197f5a311bec8bb5d 2024-11-20T21:16:45,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/40c0355cfba146dd83a6b02e4543fd0d to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/40c0355cfba146dd83a6b02e4543fd0d 2024-11-20T21:16:45,500 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/f24bb7ced89a4d1092599fe63ca1dcbd to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/f24bb7ced89a4d1092599fe63ca1dcbd 2024-11-20T21:16:45,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/4e79ec6cee8e474cae62887eae16aa48 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/4e79ec6cee8e474cae62887eae16aa48 2024-11-20T21:16:45,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/49235fb7ab6b45ffb676b0c113043c2f to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/49235fb7ab6b45ffb676b0c113043c2f 2024-11-20T21:16:45,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/2670f1dcd243477aa8fb124b75157d40 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/2670f1dcd243477aa8fb124b75157d40 2024-11-20T21:16:45,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/f5c243587493472981fb591ad9e974e2 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/f5c243587493472981fb591ad9e974e2 2024-11-20T21:16:45,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/21ac254042a34698ab99fe975f2de561 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/21ac254042a34698ab99fe975f2de561 2024-11-20T21:16:45,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/8c8fddba05324742a2577a8a5f2b7dde to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/8c8fddba05324742a2577a8a5f2b7dde 2024-11-20T21:16:45,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/48f1b09d501346faae45e3e8c1d64db6 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/48f1b09d501346faae45e3e8c1d64db6 2024-11-20T21:16:45,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/c2ee6466447444f98888c7631ca7ff37 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/c2ee6466447444f98888c7631ca7ff37 2024-11-20T21:16:45,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/3cc9626b5a72494485e9fc3c0f7119e1 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/3cc9626b5a72494485e9fc3c0f7119e1 2024-11-20T21:16:45,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/a9382244120a48c89147f8a3df6aea08, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/89fec769e2be4b538aa90fcac6ff53b3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/3ab14844bcc24834ac7ae016f5b3b17e, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/9ac3e080379241deab1f7b3ec923d256, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/289b7f3a3bf14dd09b1bc5e1006a92ef, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/bd3c4462c80e41f5b2b8e25218ab07ea, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/91e70128629646be9d03e4783b38c988, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/6d26530c4cc14bf79c7fe8554c8aa891, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/cdd671a7b0c646efac8ab8c01f67e10b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/22d9213a531448998c489da1f81abf58, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/c7d4b587c48a4f9fac3cc2c6e34a9e4d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/3488f32b3a8e48ab9fe0a5573130c98c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/8c7d879a1567425b897b7ca90a73aadc, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/0ef4640f4d6b4bdabccf244a638d251b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/7065ddc035c04b88ad0f46a43d0f7271, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/b08c581a326d4b6686ea061fb0c2849d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/b3e681715c1345babf4a8ccfcdd5c331, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/09b226dba6df4bd08abb7992c37afb4c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/2cb3c1420a274bd384472aff47b86a4d, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/5068b4cced164f5082fb70e0fbbaf2ca, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/c0be749bc380496c87bd6489e044cf64] to archive 2024-11-20T21:16:45,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T21:16:45,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/a9382244120a48c89147f8a3df6aea08 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/a9382244120a48c89147f8a3df6aea08 2024-11-20T21:16:45,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/89fec769e2be4b538aa90fcac6ff53b3 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/89fec769e2be4b538aa90fcac6ff53b3 2024-11-20T21:16:45,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/3ab14844bcc24834ac7ae016f5b3b17e to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/3ab14844bcc24834ac7ae016f5b3b17e 2024-11-20T21:16:45,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/9ac3e080379241deab1f7b3ec923d256 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/9ac3e080379241deab1f7b3ec923d256 2024-11-20T21:16:45,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/289b7f3a3bf14dd09b1bc5e1006a92ef to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/289b7f3a3bf14dd09b1bc5e1006a92ef 2024-11-20T21:16:45,512 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/bd3c4462c80e41f5b2b8e25218ab07ea to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/bd3c4462c80e41f5b2b8e25218ab07ea 2024-11-20T21:16:45,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/91e70128629646be9d03e4783b38c988 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/91e70128629646be9d03e4783b38c988 2024-11-20T21:16:45,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/6d26530c4cc14bf79c7fe8554c8aa891 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/6d26530c4cc14bf79c7fe8554c8aa891 2024-11-20T21:16:45,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/cdd671a7b0c646efac8ab8c01f67e10b to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/cdd671a7b0c646efac8ab8c01f67e10b 2024-11-20T21:16:45,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/22d9213a531448998c489da1f81abf58 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/22d9213a531448998c489da1f81abf58 2024-11-20T21:16:45,516 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/c7d4b587c48a4f9fac3cc2c6e34a9e4d to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/c7d4b587c48a4f9fac3cc2c6e34a9e4d 2024-11-20T21:16:45,516 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/3488f32b3a8e48ab9fe0a5573130c98c to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/3488f32b3a8e48ab9fe0a5573130c98c 2024-11-20T21:16:45,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/8c7d879a1567425b897b7ca90a73aadc to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/8c7d879a1567425b897b7ca90a73aadc 2024-11-20T21:16:45,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/0ef4640f4d6b4bdabccf244a638d251b to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/0ef4640f4d6b4bdabccf244a638d251b 2024-11-20T21:16:45,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/7065ddc035c04b88ad0f46a43d0f7271 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/7065ddc035c04b88ad0f46a43d0f7271 2024-11-20T21:16:45,519 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/b08c581a326d4b6686ea061fb0c2849d to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/b08c581a326d4b6686ea061fb0c2849d 2024-11-20T21:16:45,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/b3e681715c1345babf4a8ccfcdd5c331 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/b3e681715c1345babf4a8ccfcdd5c331 2024-11-20T21:16:45,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/09b226dba6df4bd08abb7992c37afb4c to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/09b226dba6df4bd08abb7992c37afb4c 2024-11-20T21:16:45,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/2cb3c1420a274bd384472aff47b86a4d to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/2cb3c1420a274bd384472aff47b86a4d 2024-11-20T21:16:45,522 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/5068b4cced164f5082fb70e0fbbaf2ca to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/5068b4cced164f5082fb70e0fbbaf2ca 2024-11-20T21:16:45,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/c0be749bc380496c87bd6489e044cf64 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/c0be749bc380496c87bd6489e044cf64 2024-11-20T21:16:45,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/2f9f320357084913be1183ad15019782, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/fc0e3fbf391f4820881047653c9e38aa, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/22a9705afcb046f082e1af32447ce711, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/e51ade5eb9c64bcfbd0c6dd068d5b8f8, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/384107f1e76f4f6890e75031d752d6e2, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/95e2863f7ca94b1e946180e6763cffed, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/beb960b23b4642709eec1ae58d75403a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/e0094382d9784eeda9756c97eb8671ec, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/0ce2be6a5d354e10a5f57babc99af44a, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/d49ef2c917ab4dc4b15d59480e47126c, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/5e9de55bc50746da9b83c828a95d988b, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/8adee7b6c70a40c68b435f0c9b339e71, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/e06bbfa0b7d542fb8262eeb97e2aa4ca, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/7f6be9eb54ab408e922cc3e1bfff3efd, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/2bf71d53c909416c8e6626b0d0c176f9, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/02fa7e13fe4643c180f85c657e5182a4, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/2543955b4f6a410cada7713e99d60aa3, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/184b788263d24e5e8e73e48e18721812, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/c3fcd37b25d249628527fe41e8324d31, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/c13f24c83eda48c0b28db51190f99d46, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/55a91c0068be41c2bc7332e7d1ae4573] to archive 2024-11-20T21:16:45,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-20T21:16:45,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/2f9f320357084913be1183ad15019782 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/2f9f320357084913be1183ad15019782 2024-11-20T21:16:45,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/fc0e3fbf391f4820881047653c9e38aa to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/fc0e3fbf391f4820881047653c9e38aa 2024-11-20T21:16:45,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/22a9705afcb046f082e1af32447ce711 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/22a9705afcb046f082e1af32447ce711 2024-11-20T21:16:45,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/e51ade5eb9c64bcfbd0c6dd068d5b8f8 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/e51ade5eb9c64bcfbd0c6dd068d5b8f8 2024-11-20T21:16:45,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/384107f1e76f4f6890e75031d752d6e2 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/384107f1e76f4f6890e75031d752d6e2 2024-11-20T21:16:45,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/95e2863f7ca94b1e946180e6763cffed to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/95e2863f7ca94b1e946180e6763cffed 2024-11-20T21:16:45,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/beb960b23b4642709eec1ae58d75403a to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/beb960b23b4642709eec1ae58d75403a 2024-11-20T21:16:45,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/e0094382d9784eeda9756c97eb8671ec to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/e0094382d9784eeda9756c97eb8671ec 2024-11-20T21:16:45,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/0ce2be6a5d354e10a5f57babc99af44a to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/0ce2be6a5d354e10a5f57babc99af44a 2024-11-20T21:16:45,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/d49ef2c917ab4dc4b15d59480e47126c to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/d49ef2c917ab4dc4b15d59480e47126c 2024-11-20T21:16:45,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/5e9de55bc50746da9b83c828a95d988b to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/5e9de55bc50746da9b83c828a95d988b 2024-11-20T21:16:45,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/8adee7b6c70a40c68b435f0c9b339e71 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/8adee7b6c70a40c68b435f0c9b339e71 2024-11-20T21:16:45,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/e06bbfa0b7d542fb8262eeb97e2aa4ca to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/e06bbfa0b7d542fb8262eeb97e2aa4ca 2024-11-20T21:16:45,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/7f6be9eb54ab408e922cc3e1bfff3efd to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/7f6be9eb54ab408e922cc3e1bfff3efd 2024-11-20T21:16:45,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/2bf71d53c909416c8e6626b0d0c176f9 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/2bf71d53c909416c8e6626b0d0c176f9 2024-11-20T21:16:45,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/02fa7e13fe4643c180f85c657e5182a4 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/02fa7e13fe4643c180f85c657e5182a4 2024-11-20T21:16:45,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/2543955b4f6a410cada7713e99d60aa3 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/2543955b4f6a410cada7713e99d60aa3 2024-11-20T21:16:45,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/184b788263d24e5e8e73e48e18721812 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/184b788263d24e5e8e73e48e18721812 2024-11-20T21:16:45,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/c3fcd37b25d249628527fe41e8324d31 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/c3fcd37b25d249628527fe41e8324d31 2024-11-20T21:16:45,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/c13f24c83eda48c0b28db51190f99d46 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/c13f24c83eda48c0b28db51190f99d46 2024-11-20T21:16:45,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/55a91c0068be41c2bc7332e7d1ae4573 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/55a91c0068be41c2bc7332e7d1ae4573 2024-11-20T21:16:45,543 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/recovered.edits/583.seqid, newMaxSeqId=583, maxSeqId=4 2024-11-20T21:16:45,544 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. 2024-11-20T21:16:45,544 DEBUG [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] regionserver.HRegion(1676): Region close journal for 2ba04bb2bd029935d159faf0939a1f26: Waiting for close lock at 1732137403840Running coprocessor pre-close hooks at 1732137403840Disabling compacts and flushes for region at 1732137403840Disabling writes for close at 1732137403840Obtaining lock to block concurrent updates at 1732137403840Preparing flush snapshotting stores in 2ba04bb2bd029935d159faf0939a1f26 at 1732137403840Finished memstore snapshotting TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1732137403840Flushing stores of TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26. at 1732137403841 (+1 ms)Flushing 2ba04bb2bd029935d159faf0939a1f26/A: creating writer at 1732137403841Flushing mob file 2ba04bb2bd029935d159faf0939a1f26/A: appending metadata at 1732137403846 (+5 ms)Flushing mob file 2ba04bb2bd029935d159faf0939a1f26/A: closing flushed file at 1732137403846Flushing 2ba04bb2bd029935d159faf0939a1f26/A: appending metadata at 1732137404255 (+409 ms)Flushing 2ba04bb2bd029935d159faf0939a1f26/A: closing flushed file at 1732137404255Flushing 2ba04bb2bd029935d159faf0939a1f26/B: creating writer at 1732137404662 (+407 ms)Flushing 2ba04bb2bd029935d159faf0939a1f26/B: appending metadata at 1732137404663 (+1 ms)Flushing 2ba04bb2bd029935d159faf0939a1f26/B: closing flushed file at 1732137404663Flushing 2ba04bb2bd029935d159faf0939a1f26/C: creating writer at 1732137405071 (+408 ms)Flushing 2ba04bb2bd029935d159faf0939a1f26/C: appending metadata at 1732137405073 (+2 ms)Flushing 2ba04bb2bd029935d159faf0939a1f26/C: closing flushed file at 1732137405073Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64a1ea53: reopening flushed file at 1732137405481 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@527c2ae9: reopening flushed file at 1732137405484 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79c2fa3e: reopening flushed file at 1732137405487 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 2ba04bb2bd029935d159faf0939a1f26 in 1650ms, sequenceid=580, compaction requested=true at 1732137405490 (+3 ms)Writing region close event to WAL at 1732137405541 (+51 ms)Running coprocessor post-close hooks at 1732137405544 (+3 ms)Closed at 1732137405544 2024-11-20T21:16:45,545 INFO [RS_CLOSE_REGION-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_REGION, pid=169}] handler.UnassignRegionHandler(157): Closed 2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:45,546 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=168 updating hbase:meta row=2ba04bb2bd029935d159faf0939a1f26, regionState=CLOSED 2024-11-20T21:16:45,547 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=169, ppid=168, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 because future has completed 2024-11-20T21:16:45,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-20T21:16:45,548 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseRegionProcedure 2ba04bb2bd029935d159faf0939a1f26, server=0b16ab3f13b7,36987,1732137236536 in 1.8600 sec 2024-11-20T21:16:45,550 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=168, resume processing ppid=167 2024-11-20T21:16:45,550 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, ppid=167, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2ba04bb2bd029935d159faf0939a1f26, UNASSIGN in 1.8640 sec 2024-11-20T21:16:45,551 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-20T21:16:45,551 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8670 sec 2024-11-20T21:16:45,552 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732137405552"}]},"ts":"1732137405552"} 2024-11-20T21:16:45,553 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-20T21:16:45,553 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-20T21:16:45,554 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8760 sec 2024-11-20T21:16:45,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-20T21:16:45,827 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:16:45,827 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-20T21:16:45,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:16:45,829 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=170, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:16:45,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-20T21:16:45,830 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=170, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:16:45,832 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:45,834 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A, FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B, FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C, FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/recovered.edits] 2024-11-20T21:16:45,835 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/2eb3a2ee7d4944e9aeaf95d1b0281404 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/2eb3a2ee7d4944e9aeaf95d1b0281404 2024-11-20T21:16:45,836 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/8b0ab4f247a3485aa819734aa52ea103 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/8b0ab4f247a3485aa819734aa52ea103 2024-11-20T21:16:45,837 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/8ed93d5118564170b2ec9cda5d7eda0c to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/A/8ed93d5118564170b2ec9cda5d7eda0c 2024-11-20T21:16:45,839 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/1f19232491654e76b1416e46dd637ebe to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/1f19232491654e76b1416e46dd637ebe 2024-11-20T21:16:45,840 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/3f232878f17f4709b1ad7c098fe23b03 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/3f232878f17f4709b1ad7c098fe23b03 2024-11-20T21:16:45,841 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/baa6ab2db1c54ba5b9d10333d43e1eee to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/B/baa6ab2db1c54ba5b9d10333d43e1eee 2024-11-20T21:16:45,843 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/35c763ea9f7244d1acffc8902ce371c7 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/35c763ea9f7244d1acffc8902ce371c7 2024-11-20T21:16:45,844 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/753f232b8d4d46eb84fe4e5615493184 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/753f232b8d4d46eb84fe4e5615493184 2024-11-20T21:16:45,845 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/a5085a556ca9420ab62e482748fd1781 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/C/a5085a556ca9420ab62e482748fd1781 2024-11-20T21:16:45,847 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/recovered.edits/583.seqid to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26/recovered.edits/583.seqid 2024-11-20T21:16:45,848 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/default/TestAcidGuarantees/2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:45,848 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-20T21:16:45,848 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T21:16:45,849 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-20T21:16:45,851 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120002b58a43bdb4756a66a65f880d6618e_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120002b58a43bdb4756a66a65f880d6618e_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:45,852 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112006180952b98043159bc92dfda02e4453_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112006180952b98043159bc92dfda02e4453_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:45,852 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200d83d20b0c98420d9e8fc19cc0d53c87_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200d83d20b0c98420d9e8fc19cc0d53c87_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:45,853 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200e8f9b13192142e8a6ccd654a93a5d26_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411200e8f9b13192142e8a6ccd654a93a5d26_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:45,854 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112016b26f810c8b443bb33bccc0fa89884a_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112016b26f810c8b443bb33bccc0fa89884a_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:45,855 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112029219ade792444dcb5825ad0c1f2c807_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112029219ade792444dcb5825ad0c1f2c807_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:45,856 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112036a20ad83fc74b2d9c8ec3eb072149a6_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112036a20ad83fc74b2d9c8ec3eb072149a6_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:45,857 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112049ebef3120094bc7a9d9bceda8b2431d_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112049ebef3120094bc7a9d9bceda8b2431d_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:45,857 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205bdd6a1c5d384c01a664b56480b2d2f7_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411205bdd6a1c5d384c01a664b56480b2d2f7_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:45,858 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206922d61236a44508b39d880593e4167c_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411206922d61236a44508b39d880593e4167c_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:45,859 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209f0bc6962bb3480299421675f0f381c9_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411209f0bc6962bb3480299421675f0f381c9_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:45,860 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a9a3af9efeba4b949ad77e48bcbd35a3_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120a9a3af9efeba4b949ad77e48bcbd35a3_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:45,861 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b5c7f043673941f99f892466c4fb467b_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b5c7f043673941f99f892466c4fb467b_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:45,862 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b7988188bfcd4a3ab4fa3120fd1aac6a_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120b7988188bfcd4a3ab4fa3120fd1aac6a_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:45,863 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c32b1676b0cf4902ad2a54132b794ed8_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120c32b1676b0cf4902ad2a54132b794ed8_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:45,863 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e4cd6d5dad4847e9afdaceccc0bcb5b4_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120e4cd6d5dad4847e9afdaceccc0bcb5b4_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:45,864 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120eb716e36a19a4dad9be51826c69412b9_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120eb716e36a19a4dad9be51826c69412b9_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:45,865 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120fe4c42c0cfbe4faa8140ca59493dda36_2ba04bb2bd029935d159faf0939a1f26 to hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241120fe4c42c0cfbe4faa8140ca59493dda36_2ba04bb2bd029935d159faf0939a1f26 2024-11-20T21:16:45,866 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-20T21:16:45,867 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=170, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:16:45,869 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-20T21:16:45,871 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-20T21:16:45,872 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=170, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:16:45,872 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-20T21:16:45,872 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732137405872"}]},"ts":"9223372036854775807"} 2024-11-20T21:16:45,874 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-20T21:16:45,874 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2ba04bb2bd029935d159faf0939a1f26, NAME => 'TestAcidGuarantees,,1732137378464.2ba04bb2bd029935d159faf0939a1f26.', STARTKEY => '', ENDKEY => ''}] 2024-11-20T21:16:45,874 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-20T21:16:45,874 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732137405874"}]},"ts":"9223372036854775807"} 2024-11-20T21:16:45,876 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-20T21:16:45,876 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=170, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-20T21:16:45,877 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 49 msec 2024-11-20T21:16:45,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39687 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-20T21:16:45,937 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-20T21:16:45,937 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-20T21:16:45,948 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251 (was 249) - Thread LEAK? -, OpenFileDescriptor=457 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=743 (was 823), ProcessCount=11 (was 11), AvailableMemoryMB=491 (was 440) - AvailableMemoryMB LEAK? - 2024-11-20T21:16:45,949 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T21:16:45,949 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T21:16:45,949 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T21:16:45,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:45,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:45,949 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T21:16:45,949 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T21:16:45,949 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1143773905, stopped=false 2024-11-20T21:16:45,950 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=0b16ab3f13b7,39687,1732137235812 2024-11-20T21:16:45,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36987-0x1005f0abc6c0001, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T21:16:45,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36987-0x1005f0abc6c0001, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:16:45,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39687-0x1005f0abc6c0000, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T21:16:45,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39687-0x1005f0abc6c0000, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:16:45,952 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T21:16:45,952 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T21:16:45,953 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T21:16:45,953 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:45,953 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '0b16ab3f13b7,36987,1732137236536' ***** 2024-11-20T21:16:45,953 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T21:16:45,953 INFO [RS:0;0b16ab3f13b7:36987 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T21:16:45,953 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36987-0x1005f0abc6c0001, quorum=127.0.0.1:57361, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T21:16:45,954 INFO [RS:0;0b16ab3f13b7:36987 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T21:16:45,954 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39687-0x1005f0abc6c0000, quorum=127.0.0.1:57361, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T21:16:45,954 INFO [RS:0;0b16ab3f13b7:36987 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T21:16:45,954 INFO [RS:0;0b16ab3f13b7:36987 {}] regionserver.HRegionServer(959): stopping server 0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:45,954 INFO [RS:0;0b16ab3f13b7:36987 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T21:16:45,954 INFO [RS:0;0b16ab3f13b7:36987 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;0b16ab3f13b7:36987. 2024-11-20T21:16:45,954 DEBUG [RS:0;0b16ab3f13b7:36987 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T21:16:45,954 DEBUG [RS:0;0b16ab3f13b7:36987 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:45,954 INFO [RS:0;0b16ab3f13b7:36987 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T21:16:45,955 INFO [RS:0;0b16ab3f13b7:36987 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T21:16:45,955 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T21:16:45,955 INFO [RS:0;0b16ab3f13b7:36987 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T21:16:45,955 INFO [RS:0;0b16ab3f13b7:36987 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T21:16:45,956 INFO [RS:0;0b16ab3f13b7:36987 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-20T21:16:45,956 DEBUG [RS:0;0b16ab3f13b7:36987 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-20T21:16:45,956 DEBUG [RS:0;0b16ab3f13b7:36987 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T21:16:45,956 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T21:16:45,956 INFO [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T21:16:45,956 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T21:16:45,956 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T21:16:45,956 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T21:16:45,956 INFO [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-20T21:16:45,986 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/hbase/meta/1588230740/.tmp/info/328bfaa96f2d4370b33be4aa5f4f3555 is 95, key is TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71./info:/1732137264408/DeleteFamily/seqid=0 2024-11-20T21:16:45,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742529_1705 (size=6514) 2024-11-20T21:16:45,993 INFO [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/hbase/meta/1588230740/.tmp/info/328bfaa96f2d4370b33be4aa5f4f3555 2024-11-20T21:16:46,010 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/hbase/meta/1588230740/.tmp/ns/59b27d6305a74ffe8455e8536417387f is 93, key is TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71./ns:/1732137264353/DeleteFamily/seqid=0 2024-11-20T21:16:46,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742530_1706 (size=5906) 2024-11-20T21:16:46,042 INFO [regionserver/0b16ab3f13b7:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T21:16:46,154 WARN [BootstrapNodeManager {}] regionserver.BootstrapNodeManager(142): failed to get live region servers from master org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=0b16ab3f13b7:39687 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:163) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] at --------Future.get--------(Unknown Source) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$RpcChannelImplementation.callMethod(AbstractRpcClient.java:628) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$Stub.getLiveRegionServers(RegionServerStatusProtos.java:17191) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncClusterConnectionImpl.getLiveRegionServers(AsyncClusterConnectionImpl.java:139) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.BootstrapNodeManager.getFromMaster(BootstrapNodeManager.java:140) ~[classes/:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:304) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 10 more 2024-11-20T21:16:46,156 DEBUG [RS:0;0b16ab3f13b7:36987 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T21:16:46,242 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-20T21:16:46,356 DEBUG [RS:0;0b16ab3f13b7:36987 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T21:16:46,414 INFO [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/hbase/meta/1588230740/.tmp/ns/59b27d6305a74ffe8455e8536417387f 2024-11-20T21:16:46,437 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/hbase/meta/1588230740/.tmp/rep_barrier/fe36142d1e144fdd92515705e0ae96a5 is 102, key is TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71./rep_barrier:/1732137264353/DeleteFamily/seqid=0 2024-11-20T21:16:46,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742531_1707 (size=6025) 2024-11-20T21:16:46,557 DEBUG [RS:0;0b16ab3f13b7:36987 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T21:16:46,566 INFO [regionserver/0b16ab3f13b7:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-20T21:16:46,566 INFO [regionserver/0b16ab3f13b7:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-20T21:16:46,757 DEBUG [RS:0;0b16ab3f13b7:36987 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-20T21:16:46,840 INFO [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/hbase/meta/1588230740/.tmp/rep_barrier/fe36142d1e144fdd92515705e0ae96a5 2024-11-20T21:16:46,860 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/hbase/meta/1588230740/.tmp/table/1311b2f9b6b64b709cbdbd19d2d2d532 is 96, key is TestAcidGuarantees,,1732137239659.f3e2abb3d075e2d6a5c2948af99b8c71./table:/1732137264353/DeleteFamily/seqid=0 2024-11-20T21:16:46,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742532_1708 (size=5936) 2024-11-20T21:16:46,863 INFO [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/hbase/meta/1588230740/.tmp/table/1311b2f9b6b64b709cbdbd19d2d2d532 2024-11-20T21:16:46,866 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/hbase/meta/1588230740/.tmp/info/328bfaa96f2d4370b33be4aa5f4f3555 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/hbase/meta/1588230740/info/328bfaa96f2d4370b33be4aa5f4f3555 2024-11-20T21:16:46,869 INFO [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/hbase/meta/1588230740/info/328bfaa96f2d4370b33be4aa5f4f3555, entries=12, sequenceid=90, filesize=6.4 K 2024-11-20T21:16:46,870 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/hbase/meta/1588230740/.tmp/ns/59b27d6305a74ffe8455e8536417387f as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/hbase/meta/1588230740/ns/59b27d6305a74ffe8455e8536417387f 2024-11-20T21:16:46,872 INFO [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/hbase/meta/1588230740/ns/59b27d6305a74ffe8455e8536417387f, entries=8, sequenceid=90, filesize=5.8 K 2024-11-20T21:16:46,873 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/hbase/meta/1588230740/.tmp/rep_barrier/fe36142d1e144fdd92515705e0ae96a5 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/hbase/meta/1588230740/rep_barrier/fe36142d1e144fdd92515705e0ae96a5 2024-11-20T21:16:46,876 INFO [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/hbase/meta/1588230740/rep_barrier/fe36142d1e144fdd92515705e0ae96a5, entries=6, sequenceid=90, filesize=5.9 K 2024-11-20T21:16:46,876 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/hbase/meta/1588230740/.tmp/table/1311b2f9b6b64b709cbdbd19d2d2d532 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/hbase/meta/1588230740/table/1311b2f9b6b64b709cbdbd19d2d2d532 2024-11-20T21:16:46,880 INFO [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/hbase/meta/1588230740/table/1311b2f9b6b64b709cbdbd19d2d2d532, entries=7, sequenceid=90, filesize=5.8 K 2024-11-20T21:16:46,880 INFO [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 924ms, sequenceid=90, compaction requested=false 2024-11-20T21:16:46,884 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-20T21:16:46,885 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T21:16:46,885 INFO [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T21:16:46,885 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732137405956Running coprocessor pre-close hooks at 1732137405956Disabling compacts and flushes for region at 1732137405956Disabling writes for close at 1732137405956Obtaining lock to block concurrent updates at 1732137405956Preparing flush snapshotting stores in 1588230740 at 1732137405956Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732137405957 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732137405957Flushing 1588230740/info: creating writer at 1732137405957Flushing 1588230740/info: appending metadata at 1732137405986 (+29 ms)Flushing 1588230740/info: closing flushed file at 1732137405986Flushing 1588230740/ns: creating writer at 1732137405996 (+10 ms)Flushing 1588230740/ns: appending metadata at 1732137406010 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732137406010Flushing 1588230740/rep_barrier: creating writer at 1732137406417 (+407 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732137406436 (+19 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732137406437 (+1 ms)Flushing 1588230740/table: creating writer at 1732137406843 (+406 ms)Flushing 1588230740/table: appending metadata at 1732137406859 (+16 ms)Flushing 1588230740/table: closing flushed file at 1732137406859Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e5f8898: reopening flushed file at 1732137406866 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6adfc8b1: reopening flushed file at 1732137406870 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f05fc81: reopening flushed file at 1732137406872 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@13d5664e: reopening flushed file at 1732137406876 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 924ms, sequenceid=90, compaction requested=false at 1732137406880 (+4 ms)Writing region close event to WAL at 1732137406882 (+2 ms)Running coprocessor post-close hooks at 1732137406885 (+3 ms)Closed at 1732137406885 2024-11-20T21:16:46,885 DEBUG [RS_CLOSE_META-regionserver/0b16ab3f13b7:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T21:16:46,957 INFO [RS:0;0b16ab3f13b7:36987 {}] regionserver.HRegionServer(976): stopping server 0b16ab3f13b7,36987,1732137236536; all regions closed. 2024-11-20T21:16:46,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741834_1010 (size=25344) 2024-11-20T21:16:46,963 DEBUG [RS:0;0b16ab3f13b7:36987 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/oldWALs 2024-11-20T21:16:46,963 INFO [RS:0;0b16ab3f13b7:36987 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0b16ab3f13b7%2C36987%2C1732137236536.meta:.meta(num 1732137239253) 2024-11-20T21:16:46,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741833_1009 (size=22366984) 2024-11-20T21:16:46,966 DEBUG [RS:0;0b16ab3f13b7:36987 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/oldWALs 2024-11-20T21:16:46,966 INFO [RS:0;0b16ab3f13b7:36987 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 0b16ab3f13b7%2C36987%2C1732137236536:(num 1732137238697) 2024-11-20T21:16:46,966 DEBUG [RS:0;0b16ab3f13b7:36987 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T21:16:46,966 INFO [RS:0;0b16ab3f13b7:36987 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T21:16:46,966 INFO [RS:0;0b16ab3f13b7:36987 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T21:16:46,967 INFO [RS:0;0b16ab3f13b7:36987 {}] hbase.ChoreService(370): Chore service for: regionserver/0b16ab3f13b7:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-20T21:16:46,967 INFO [RS:0;0b16ab3f13b7:36987 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T21:16:46,967 INFO [regionserver/0b16ab3f13b7:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T21:16:46,968 INFO [RS:0;0b16ab3f13b7:36987 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36987 2024-11-20T21:16:46,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36987-0x1005f0abc6c0001, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/0b16ab3f13b7,36987,1732137236536 2024-11-20T21:16:46,972 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39687-0x1005f0abc6c0000, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T21:16:46,972 INFO [RS:0;0b16ab3f13b7:36987 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T21:16:46,975 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [0b16ab3f13b7,36987,1732137236536] 2024-11-20T21:16:46,977 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/0b16ab3f13b7,36987,1732137236536 already deleted, retry=false 2024-11-20T21:16:46,977 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 0b16ab3f13b7,36987,1732137236536 expired; onlineServers=0 2024-11-20T21:16:46,977 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '0b16ab3f13b7,39687,1732137235812' ***** 2024-11-20T21:16:46,977 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T21:16:46,977 INFO [M:0;0b16ab3f13b7:39687 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T21:16:46,977 INFO [M:0;0b16ab3f13b7:39687 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T21:16:46,977 DEBUG [M:0;0b16ab3f13b7:39687 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T21:16:46,977 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T21:16:46,977 DEBUG [M:0;0b16ab3f13b7:39687 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T21:16:46,977 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster-HFileCleaner.small.0-1732137238266 {}] cleaner.HFileCleaner(306): Exit Thread[master/0b16ab3f13b7:0:becomeActiveMaster-HFileCleaner.small.0-1732137238266,5,FailOnTimeoutGroup] 2024-11-20T21:16:46,978 DEBUG [master/0b16ab3f13b7:0:becomeActiveMaster-HFileCleaner.large.0-1732137238258 {}] cleaner.HFileCleaner(306): Exit Thread[master/0b16ab3f13b7:0:becomeActiveMaster-HFileCleaner.large.0-1732137238258,5,FailOnTimeoutGroup] 2024-11-20T21:16:46,978 INFO [M:0;0b16ab3f13b7:39687 {}] hbase.ChoreService(370): Chore service for: master/0b16ab3f13b7:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T21:16:46,978 INFO [M:0;0b16ab3f13b7:39687 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T21:16:46,978 DEBUG [M:0;0b16ab3f13b7:39687 {}] master.HMaster(1795): Stopping service threads 2024-11-20T21:16:46,978 INFO [M:0;0b16ab3f13b7:39687 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T21:16:46,978 INFO [M:0;0b16ab3f13b7:39687 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T21:16:46,978 ERROR [M:0;0b16ab3f13b7:39687 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[IPC Client (2068722025) connection to localhost/127.0.0.1:35549 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:35549,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-20T21:16:46,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39687-0x1005f0abc6c0000, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T21:16:46,980 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39687-0x1005f0abc6c0000, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T21:16:46,981 INFO [M:0;0b16ab3f13b7:39687 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T21:16:46,981 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T21:16:46,983 DEBUG [M:0;0b16ab3f13b7:39687 {}] zookeeper.ZKUtil(347): master:39687-0x1005f0abc6c0000, quorum=127.0.0.1:57361, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T21:16:46,983 WARN [M:0;0b16ab3f13b7:39687 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T21:16:46,983 INFO [M:0;0b16ab3f13b7:39687 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/.lastflushedseqids 2024-11-20T21:16:46,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742533_1709 (size=72) 2024-11-20T21:16:47,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36987-0x1005f0abc6c0001, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T21:16:47,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36987-0x1005f0abc6c0001, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T21:16:47,076 INFO [RS:0;0b16ab3f13b7:36987 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T21:16:47,076 INFO [RS:0;0b16ab3f13b7:36987 {}] regionserver.HRegionServer(1031): Exiting; stopping=0b16ab3f13b7,36987,1732137236536; zookeeper connection closed. 2024-11-20T21:16:47,077 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@57a2c0d {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@57a2c0d 2024-11-20T21:16:47,077 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-20T21:16:47,397 INFO [M:0;0b16ab3f13b7:39687 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T21:16:47,397 INFO [M:0;0b16ab3f13b7:39687 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T21:16:47,397 DEBUG [M:0;0b16ab3f13b7:39687 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T21:16:47,397 INFO [M:0;0b16ab3f13b7:39687 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T21:16:47,398 DEBUG [M:0;0b16ab3f13b7:39687 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T21:16:47,398 DEBUG [M:0;0b16ab3f13b7:39687 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T21:16:47,398 DEBUG [M:0;0b16ab3f13b7:39687 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T21:16:47,398 INFO [M:0;0b16ab3f13b7:39687 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=775.78 KB heapSize=950.59 KB 2024-11-20T21:16:47,413 DEBUG [M:0;0b16ab3f13b7:39687 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/64656bf0f3764514978f4ed3bb223a5c is 82, key is hbase:meta,,1/info:regioninfo/1732137239336/Put/seqid=0 2024-11-20T21:16:47,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742534_1710 (size=5672) 2024-11-20T21:16:47,816 INFO [M:0;0b16ab3f13b7:39687 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2156 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/64656bf0f3764514978f4ed3bb223a5c 2024-11-20T21:16:47,845 DEBUG [M:0;0b16ab3f13b7:39687 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3770e13b554d482ab6c4d4a663a664bc is 2479, key is \x00\x00\x00\x00\x00\x00\x00\x93/proc:d/1732137381547/Put/seqid=0 2024-11-20T21:16:47,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742535_1711 (size=45352) 2024-11-20T21:16:48,250 INFO [M:0;0b16ab3f13b7:39687 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=775.23 KB at sequenceid=2156 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3770e13b554d482ab6c4d4a663a664bc 2024-11-20T21:16:48,254 INFO [M:0;0b16ab3f13b7:39687 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3770e13b554d482ab6c4d4a663a664bc 2024-11-20T21:16:48,269 DEBUG [M:0;0b16ab3f13b7:39687 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1b90d2a18b784fb9ac0dc37e525a5793 is 69, key is 0b16ab3f13b7,36987,1732137236536/rs:state/1732137238400/Put/seqid=0 2024-11-20T21:16:48,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073742536_1712 (size=5156) 2024-11-20T21:16:48,673 INFO [M:0;0b16ab3f13b7:39687 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2156 (bloomFilter=true), to=hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1b90d2a18b784fb9ac0dc37e525a5793 2024-11-20T21:16:48,677 DEBUG [M:0;0b16ab3f13b7:39687 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/64656bf0f3764514978f4ed3bb223a5c as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/64656bf0f3764514978f4ed3bb223a5c 2024-11-20T21:16:48,681 INFO [M:0;0b16ab3f13b7:39687 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/64656bf0f3764514978f4ed3bb223a5c, entries=8, sequenceid=2156, filesize=5.5 K 2024-11-20T21:16:48,681 DEBUG [M:0;0b16ab3f13b7:39687 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3770e13b554d482ab6c4d4a663a664bc as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3770e13b554d482ab6c4d4a663a664bc 2024-11-20T21:16:48,685 INFO [M:0;0b16ab3f13b7:39687 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3770e13b554d482ab6c4d4a663a664bc 2024-11-20T21:16:48,685 INFO [M:0;0b16ab3f13b7:39687 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3770e13b554d482ab6c4d4a663a664bc, entries=170, sequenceid=2156, filesize=44.3 K 2024-11-20T21:16:48,686 DEBUG [M:0;0b16ab3f13b7:39687 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1b90d2a18b784fb9ac0dc37e525a5793 as hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1b90d2a18b784fb9ac0dc37e525a5793 2024-11-20T21:16:48,689 INFO [M:0;0b16ab3f13b7:39687 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35549/user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1b90d2a18b784fb9ac0dc37e525a5793, entries=1, sequenceid=2156, filesize=5.0 K 2024-11-20T21:16:48,689 INFO [M:0;0b16ab3f13b7:39687 {}] regionserver.HRegion(3140): Finished flush of dataSize ~775.78 KB/794400, heapSize ~950.30 KB/973104, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1291ms, sequenceid=2156, compaction requested=false 2024-11-20T21:16:48,693 INFO [M:0;0b16ab3f13b7:39687 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T21:16:48,693 DEBUG [M:0;0b16ab3f13b7:39687 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732137407397Disabling compacts and flushes for region at 1732137407397Disabling writes for close at 1732137407398 (+1 ms)Obtaining lock to block concurrent updates at 1732137407398Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732137407398Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=794400, getHeapSize=973344, getOffHeapSize=0, getCellsCount=2351 at 1732137407398Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732137407398Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732137407399 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732137407412 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732137407413 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732137407820 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732137407844 (+24 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732137407844Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732137408254 (+410 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732137408268 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732137408268Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20232edc: reopening flushed file at 1732137408677 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7516ccd8: reopening flushed file at 1732137408681 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12b761b6: reopening flushed file at 1732137408685 (+4 ms)Finished flush of dataSize ~775.78 KB/794400, heapSize ~950.30 KB/973104, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1291ms, sequenceid=2156, compaction requested=false at 1732137408689 (+4 ms)Writing region close event to WAL at 1732137408693 (+4 ms)Closed at 1732137408693 2024-11-20T21:16:48,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42371 is added to blk_1073741830_1006 (size=935739) 2024-11-20T21:16:48,697 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/6a8a2151-639e-01e7-f274-3c5f0c618094/MasterData/WALs/0b16ab3f13b7,39687,1732137235812/0b16ab3f13b7%2C39687%2C1732137235812.1732137237546 not finished, retry = 0 2024-11-20T21:16:48,798 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T21:16:48,798 INFO [M:0;0b16ab3f13b7:39687 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-20T21:16:48,798 INFO [M:0;0b16ab3f13b7:39687 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:39687 2024-11-20T21:16:48,798 INFO [M:0;0b16ab3f13b7:39687 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T21:16:48,902 INFO [M:0;0b16ab3f13b7:39687 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T21:16:48,902 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39687-0x1005f0abc6c0000, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T21:16:48,903 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39687-0x1005f0abc6c0000, quorum=127.0.0.1:57361, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T21:16:48,911 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7acb84b5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T21:16:48,913 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6e1ebae6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T21:16:48,914 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T21:16:48,914 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T21:16:48,914 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42946cd0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7/hadoop.log.dir/,STOPPED} 2024-11-20T21:16:48,918 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-20T21:16:48,918 WARN [BP-394376-172.17.0.3-1732137232539 heartbeating to localhost/127.0.0.1:35549 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-20T21:16:48,918 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-20T21:16:48,918 WARN [BP-394376-172.17.0.3-1732137232539 heartbeating to localhost/127.0.0.1:35549 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-394376-172.17.0.3-1732137232539 (Datanode Uuid 9cda7d19-7c5a-4207-a46c-89245f61bcba) service to localhost/127.0.0.1:35549 2024-11-20T21:16:48,921 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7/cluster_6fcb2757-7fac-998c-5e18-650f39eceb78/data/data1/current/BP-394376-172.17.0.3-1732137232539 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T21:16:48,921 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7/cluster_6fcb2757-7fac-998c-5e18-650f39eceb78/data/data2/current/BP-394376-172.17.0.3-1732137232539 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-20T21:16:48,922 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-20T21:16:48,931 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T21:16:48,932 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-20T21:16:48,932 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-20T21:16:48,932 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-20T21:16:48,932 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/891e1ffb-3274-54ca-18d7-5742b1b558f7/hadoop.log.dir/,STOPPED} 2024-11-20T21:16:48,956 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-20T21:16:49,102 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down